def add_prediction_op(self): """Runs an rnn on the input using TensorFlows's @tf.nn.dynamic_rnn function, and returns the final state as a prediction. TODO: - Call tf.nn.dynamic_rnn using @cell below. See: https://www.tensorflow.org/api_docs/python/nn/recurrent_neural_networks - Apply a sigmoid transformation on the final state to normalize the inputs between 0 and 1. Returns: preds: tf.Tensor of shape (batch_size, 1) """ # Pick out the cell to use here. if self.config.cell == "rnn": cell = RNNCell(1, 1) elif self.config.cell == "gru": cell = GRUCell(1, 1) elif self.config.cell == "lstm": cell = tf.nn.rnn_cell.LSTMCell(1) else: raise ValueError("Unsupported cell type.") x = self.inputs_placeholder ### YOUR CODE HERE (~2-3 lines) outputs, state = tf.nn.dynamic_rnn(cell, self.inputs_placeholder, initial_state= cell.zero_state([cell.state_size,cell.state_size], dtype=tf.float32), dtype=tf.float32) preds = tf.sigmoid(outputs) ### END YOUR CODE return preds #state # preds
def add_prediction_op(self): """Adds the unrolled RNN: h_0 = 0 for t in 1 to T: o_t, h_t = cell(x_t, h_{t-1}) o_drop_t = Dropout(o_t, dropout_rate) y_t = o_drop_t U + b_2 TODO: There a quite a few things you'll need to do in this function: - Define the variables U, b_2. - Define the vector h as a constant and inititalize it with zeros. See tf.zeros and tf.shape for information on how to initialize this variable to be of the right shape. https://www.tensorflow.org/api_docs/python/constant_op/constant_value_tensors#zeros https://www.tensorflow.org/api_docs/python/array_ops/shapes_and_shaping#shape - In a for loop, begin to unroll the RNN sequence. Collect the predictions in a list. - When unrolling the loop, from the second iteration onwards, you will HAVE to call tf.get_variable_scope().reuse_variables() so that you do not create new variables in the RNN cell. See https://www.tensorflow.org/versions/master/how_tos/variable_scope/ - Concatenate and reshape the predictions into a predictions tensor. Hint: You will find the function tf.pack (similar to np.asarray) useful to assemble a list of tensors into a larger tensor. https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#pack Hint: You will find the function tf.transpose and the perms argument useful to shuffle the indices of the tensor. https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#transpose Remember: * Use the xavier initilization for matrices. * Note that tf.nn.dropout takes the keep probability (1 - p_drop) as an argument. The keep probability should be set to the value of self.dropout_placeholder Returns: pred: tf.Tensor of shape (batch_size, max_length, n_classes) """ x = self.add_embedding() dropout_rate = self.dropout_placeholder preds = [] # Predicted output at each timestep should go here! # Use the cell defined below. For Q2, we will just be using the # RNNCell you defined, but for Q3, we will run this code again # with a GRU cell! if self.config.cell == "rnn": cell = RNNCell(Config.n_features * Config.embed_size, Config.hidden_size) elif self.config.cell == "gru": cell = GRUCell(Config.n_features * Config.embed_size, Config.hidden_size) elif self.config.cell == "mlstm": cell = MultiplicativeLSTMCell( Config.n_features * Config.embed_size, Config.hidden_size) elif self.config.cell == "mgru": cell = MultiplicativeGRUCell(Config.n_features * Config.embed_size, Config.hidden_size) else: raise ValueError("Unsuppported cell type: " + self.config.cell) # Define U and b2 as variables. # Initialize state as vector of zeros. initializer = tf.contrib.layers.xavier_initializer() U = tf.get_variable("U", shape=(Config.hidden_size, Config.n_classes), initializer=initializer) b2 = tf.get_variable("b2", shape=(Config.n_classes, )) h = cell.zero_state(batch_size=tf.shape(x)[0], dtype=tf.float32) with tf.variable_scope("RNN"): for time_step in range(self.max_length): if time_step > 0: tf.get_variable_scope().reuse_variables() o, h = cell(x[:, time_step, :], h) o_drop = tf.nn.dropout(o, dropout_rate) y = tf.matmul(o_drop, U) + tf.expand_dims(b2, 0) preds.append(y) # Make sure to reshape @preds here. preds = tf.stack(preds, axis=1) assert preds.get_shape().as_list() == [ None, self.max_length, self.config.n_classes ], "predictions are not of the right shape. Expected {}, got {}".format( [None, self.max_length, self.config.n_classes], preds.get_shape().as_list()) return preds