def add_prediction_op(self):
"""Runs an rnn on the input using TensorFlows's
@tf.nn.dynamic_rnn function, and returns the final state as a prediction.
TODO:
- Call tf.nn.dynamic_rnn using @cell below. See:
https://www.tensorflow.org/api_docs/python/nn/recurrent_neural_networks
- Apply a sigmoid transformation on the final state to
normalize the inputs between 0 and 1.
Returns:
preds: tf.Tensor of shape (batch_size, 1)
"""
# Pick out the cell to use here.
if self.config.cell == "rnn":
cell = RNNCell(1, 1)
elif self.config.cell == "gru":
cell = GRUCell(1, 1)
elif self.config.cell == "lstm":
cell = tf.nn.rnn_cell.LSTMCell(1)
else:
raise ValueError("Unsupported cell type.")
x = self.inputs_placeholder
### YOUR CODE HERE (~2-3 lines)
outputs, state = tf.nn.dynamic_rnn(cell, self.inputs_placeholder,
initial_state= cell.zero_state([cell.state_size,cell.state_size], dtype=tf.float32),
dtype=tf.float32)
preds = tf.sigmoid(outputs)
### END YOUR CODE
return preds #state # preds
def add_prediction_op(self):
"""Adds the unrolled RNN:
h_0 = 0
for t in 1 to T:
o_t, h_t = cell(x_t, h_{t-1})
o_drop_t = Dropout(o_t, dropout_rate)
y_t = o_drop_t U + b_2
TODO: There a quite a few things you'll need to do in this function:
- Define the variables U, b_2.
- Define the vector h as a constant and inititalize it with
zeros. See tf.zeros and tf.shape for information on how
to initialize this variable to be of the right shape.
https://www.tensorflow.org/api_docs/python/constant_op/constant_value_tensors#zeros
https://www.tensorflow.org/api_docs/python/array_ops/shapes_and_shaping#shape
- In a for loop, begin to unroll the RNN sequence. Collect
the predictions in a list.
- When unrolling the loop, from the second iteration
onwards, you will HAVE to call
tf.get_variable_scope().reuse_variables() so that you do
not create new variables in the RNN cell.
See https://www.tensorflow.org/versions/master/how_tos/variable_scope/
- Concatenate and reshape the predictions into a predictions
tensor.
Hint: You will find the function tf.pack (similar to np.asarray)
useful to assemble a list of tensors into a larger tensor.
https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#pack
Hint: You will find the function tf.transpose and the perms
argument useful to shuffle the indices of the tensor.
https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#transpose
Remember:
* Use the xavier initilization for matrices.
* Note that tf.nn.dropout takes the keep probability (1 - p_drop) as an argument.
The keep probability should be set to the value of self.dropout_placeholder
Returns:
pred: tf.Tensor of shape (batch_size, max_length, n_classes)
"""
x = self.add_embedding()
dropout_rate = self.dropout_placeholder
preds = [] # Predicted output at each timestep should go here!
# Use the cell defined below. For Q2, we will just be using the
# RNNCell you defined, but for Q3, we will run this code again
# with a GRU cell!
if self.config.cell == "rnn":
cell = RNNCell(Config.n_features * Config.embed_size,
Config.hidden_size)
elif self.config.cell == "gru":
cell = GRUCell(Config.n_features * Config.embed_size,
Config.hidden_size)
elif self.config.cell == "mlstm":
cell = MultiplicativeLSTMCell(
Config.n_features * Config.embed_size, Config.hidden_size)
elif self.config.cell == "mgru":
cell = MultiplicativeGRUCell(Config.n_features * Config.embed_size,
Config.hidden_size)
else:
raise ValueError("Unsuppported cell type: " + self.config.cell)
# Define U and b2 as variables.
# Initialize state as vector of zeros.
initializer = tf.contrib.layers.xavier_initializer()
U = tf.get_variable("U",
shape=(Config.hidden_size, Config.n_classes),
initializer=initializer)
b2 = tf.get_variable("b2", shape=(Config.n_classes, ))
h = cell.zero_state(batch_size=tf.shape(x)[0], dtype=tf.float32)
with tf.variable_scope("RNN"):
for time_step in range(self.max_length):
if time_step > 0:
tf.get_variable_scope().reuse_variables()
o, h = cell(x[:, time_step, :], h)
o_drop = tf.nn.dropout(o, dropout_rate)
y = tf.matmul(o_drop, U) + tf.expand_dims(b2, 0)
preds.append(y)
# Make sure to reshape @preds here.
preds = tf.stack(preds, axis=1)
assert preds.get_shape().as_list() == [
None, self.max_length, self.config.n_classes
], "predictions are not of the right shape. Expected {}, got {}".format(
[None, self.max_length, self.config.n_classes],
preds.get_shape().as_list())
return preds