def compute_cell_dynamics(args):
with tf.Graph().as_default():
# You can change this around, but make sure to reset it to 41 when
# submitting.
np.random.seed(41)
tf.set_random_seed(41)
with tf.variable_scope("dynamics"):
x_placeholder = tf.placeholder(tf.float32, shape=(None,1))
h_placeholder = tf.placeholder(tf.float32, shape=(None,1))
def mat(x):
return np.atleast_2d(np.array(x, dtype=np.float32))
def vec(x):
return np.atleast_1d(np.array(x, dtype=np.float32))
with tf.variable_scope("cell"):
Ur, Wr, Uz, Wz, Uo, Wo = [mat(3*x) for x in np.random.randn(6)]
br, bz, bo = [vec(x) for x in np.random.randn(3)]
params = [Ur, Wr, br, Uz, Wz, bz, Uo, Wo, bo]
tf.get_variable("U_r", initializer=Ur)
tf.get_variable("W_r", initializer=Wr)
tf.get_variable("b_r", initializer=br)
tf.get_variable("U_z", initializer=Uz)
tf.get_variable("W_z", initializer=Wz)
tf.get_variable("b_z", initializer=bz)
tf.get_variable("U_o", initializer=Uo)
tf.get_variable("W_o", initializer=Wo)
tf.get_variable("b_o", initializer=bo)
tf.get_variable_scope().reuse_variables()
y_gru, h_gru = GRUCell(1,1)(x_placeholder, h_placeholder, scope="cell")
y_rnn, h_rnn = GRUCell(1,1)(x_placeholder, h_placeholder, scope="cell")
init = tf.global_variables_initializer()
with tf.Session() as session:
session.run(init)
x = mat(np.zeros(1000)).T
h = mat(np.linspace(-3, 3, 1000)).T
ht_gru = session.run([h_gru], feed_dict={x_placeholder: x, h_placeholder: h})
ht_rnn = session.run([h_rnn], feed_dict={x_placeholder: x, h_placeholder: h})
ht_gru = np.array(ht_gru)[0]
ht_rnn = np.array(ht_rnn)[0]
make_dynamics_plot(args, 0, h, ht_rnn, ht_gru, params)
x = mat(np.ones(1000)).T
h = mat(np.linspace(-3, 3, 1000)).T
ht_gru = session.run([h_gru], feed_dict={x_placeholder: x, h_placeholder: h})
ht_rnn = session.run([h_rnn], feed_dict={x_placeholder: x, h_placeholder: h})
ht_gru = np.array(ht_gru)[0]
ht_rnn = np.array(ht_rnn)[0]
make_dynamics_plot(args, 1, h, ht_rnn, ht_gru, params)
def add_prediction_op(self):
"""Runs an rnn on the input using TensorFlows's
@tf.nn.dynamic_rnn function, and returns the final state as a prediction.
TODO:
- Call tf.nn.dynamic_rnn using @cell below. See:
https://www.tensorflow.org/api_docs/python/nn/recurrent_neural_networks
- Apply a sigmoid transformation on the final state to
normalize the inputs between 0 and 1.
Returns:
preds: tf.Tensor of shape (batch_size, 1)
"""
# Pick out the cell to use here.
if self.config.cell == "rnn":
cell = RNNCell(1, 1)
elif self.config.cell == "gru":
cell = GRUCell(1, 1)
elif self.config.cell == "lstm":
cell = tf.nn.rnn_cell.LSTMCell(1)
else:
raise ValueError("Unsupported cell type.")
x = self.inputs_placeholder
### YOUR CODE HERE (~2-3 lines)
_, state = tf.nn.dynamic_rnn(cell, x, dtype=tf.float32)
preds = tf.nn.sigmoid(state)
### END YOUR CODE
return preds #state # preds
def add_prediction_op(self):
"""Runs an rnn on the input
Returns:
preds: tf.Tensor of shape (batch_size, 1)
"""
# Pick out the cell to use here.
if self.config.cell == "rnn":
cell = RNNCell(1, 1)
elif self.config.cell == "gru":
cell = GRUCell(1, 1)
elif self.config.cell == "lstm":
cell = tf.nn.rnn_cell.LSTMCell(1)
else:
raise ValueError("Unsupported cell type.")
x = self.inputs_placeholder
preds, _ = tf.nn.dynamic_rnn(cell, x, dtype = tf.float32)
preds = tf.transpose(preds, [1, 0, 2])
preds = tf.sigmoid(preds[preds.shape[0]-1])
return preds
def __init__(self, input_size, hidden_size, output_size):
super(GRUModel, self).__init__()
self.hidden_size = hidden_size
# Our own GRU implementation
self.gru = GRUCell(input_size, hidden_size)
# Fully-connected output layer
self.fc = torch.nn.Linear(hidden_size, output_size)
def add_prediction_op(self):
"""
Returns:
pred: tf.Tensor of shape (batch_size, max_length, n_classes)
"""
x = tf.cast(self.add_embedding(), tf.float32)
cell = GRUCell(self.config.embedding_size, self.config.hidden_size)
U = tf.get_variable(
'U',
shape=[self.config.hidden_size, self.config.n_classes],
initializer=tf.contrib.layers.xavier_initializer())
b2 = tf.get_variable('b2',
shape=[
self.config.n_classes,
],
initializer=tf.constant_initializer(0))
h_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size])
preds = list()
with tf.variable_scope('RNN'):
for time_step in range(self.config.max_sentence_length):
if time_step > 0:
tf.get_variable_scope().reuse_variables()
o_t, h_t = cell(x[:, time_step, :], h_t)
o_drop_t = tf.nn.dropout(o_t, self.dropout_placeholder)
y_t = tf.matmul(o_drop_t, U) + b2
preds.append(y_t)
preds = tf.stack(preds, 1) # converts from list to tensor
assert preds.get_shape().as_list() == [
None, self.config.max_sentence_length, self.config.n_classes
], 'predictions are not of the right shape. Expected {}, got {}'.format(
[None, self.max_length, self.config.n_classes],
preds.get_shape().as_list())
return preds
def add_prediction_op(self):
"""Adds the unrolled RNN:
h_0 = 0
for t in 1 to T:
o_t, h_t = cell(x_t, h_{t-1})
o_drop_t = Dropout(o_t, dropout_rate)
y_t = o_drop_t U + b_2
TODO: There a quite a few things you'll need to do in this function:
- Define the variables U, b_2.
- Define the vector h as a constant and inititalize it with
zeros. See tf.zeros and tf.shape for information on how
to initialize this variable to be of the right shape.
https://www.tensorflow.org/api_docs/python/constant_op/constant_value_tensors#zeros
https://www.tensorflow.org/api_docs/python/array_ops/shapes_and_shaping#shape
- In a for loop, begin to unroll the RNN sequence. Collect
the predictions in a list.
- When unrolling the loop, from the second iteration
onwards, you will HAVE to call
tf.get_variable_scope().reuse_variables() so that you do
not create new variables in the RNN cell.
See https://www.tensorflow.org/versions/master/how_tos/variable_scope/
- Concatenate and reshape the predictions into a predictions
tensor.
Hint: You will find the function tf.pack (similar to np.asarray)
useful to assemble a list of tensors into a larger tensor.
https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#pack
Hint: You will find the function tf.transpose and the perms
argument useful to shuffle the indices of the tensor.
https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#transpose
Remember:
* Use the xavier initilization for matrices.
* Note that tf.nn.dropout takes the keep probability (1 - p_drop) as an argument.
The keep probability should be set to the value of self.dropout_placeholder
Returns:
pred: tf.Tensor of shape (batch_size, max_length, n_classes)
"""
x = self.add_embedding()
dropout_rate = self.dropout_placeholder
if Config.cnn:
repr_ = tf.expand_dims(tf.transpose(x, perm=[0, 2, 1]), -1)
cnn_output1 = self.cnn_layer(repr_, Config.stepsize[0])
cnn_output2 = self.cnn_layer(repr_, Config.stepsize[1])
#cnn_output1_trunc = tf.slice(cnn_output1,[0,]*3,[-1,(self.max_length-Config.stepsize[1]+1)//2, Config.m])
#assert cnn_output1_trunc.get_shape().as_list() == [None, (self.max_length-Config.stepsize[1]+1)//2, Config.m], "truncated results are not of the right shape. Expected {}, got {}".format([None, (dim1-Config.step_size[1]+1)//2, self.config.m], cnn_output1_trunc.get_shape().as_list())
x = tf.concat([x, cnn_output1, cnn_output2], 2)
#x = self.fully_connected_layer(x)
preds = [] # Predicted output at each timestep should go here!
# Use the cell defined below. For Q2, we will just be using the
# RNNCell you defined, but for Q3, we will run this code again
# with a GRU cell!
if self.config.cell == "rnn":
cell = RNNCell(Config.n_features * Config.embed_size,
Config.hidden_size)
elif self.config.cell == "gru":
cell = GRUCell(Config.n_features * Config.embed_size,
Config.hidden_size)
elif self.config.cell == "lstm":
rnn_inputs = tf.nn.dropout(x, dropout_rate)
lstm_features = self.bilstm_layer(rnn_inputs)
preds = self.project_layer(lstm_features, Config.n_classes)
return preds
else:
raise ValueError("Unsuppported cell type: " + self.config.cell)
# Define U and b2 as variables.
# Initialize state as vector of zeros.
### YOUR CODE HERE (~4-6 lines)
self.U = tf.get_variable(
'U', [Config.hidden_size, Config.n_classes],
initializer=tf.contrib.layers.xavier_initializer())
self.b2 = tf.get_variable(
'b2', [
Config.n_classes,
],
initializer=tf.contrib.layers.xavier_initializer())
h = tf.zeros([tf.shape(x)[0], Config.hidden_size])
### END YOUR CODE
with tf.variable_scope("RNN"):
for time_step in range(self.max_length):
### YOUR CODE HERE (~6-10 lines)
if time_step >= 1:
tf.get_variable_scope().reuse_variables()
o, h = cell(x[:, time_step, :], h)
o_drop = tf.nn.dropout(o, dropout_rate)
curPred = tf.matmul(o_drop, self.U) + self.b2
preds.append(curPred)
### END YOUR CODE
# Make sure to reshape @preds here.
### YOUR CODE HERE (~2-4 lines)
preds = tf.stack(preds)
preds = tf.transpose(preds, perm=[1, 0, 2])
preds = tf.reshape(preds, [-1, Config.max_length, Config.n_classes])
### END YOUR CODE
assert preds.get_shape().as_list() == [
None, self.max_length, Config.n_classes
], "predictions are not of the right shape. Expected {}, got {}".format(
[None, self.max_length, self.config.n_classes],
preds.get_shape().as_list())
return preds
def add_prediction_op(self):
"""Adds the unrolled RNN:
h_0 = 0
for t in 1 to T:
o_t, h_t = cell(x_t, h_{t-1})
o_drop_t = Dropout(o_t, dropout_rate)
y_t = o_drop_t U + b_2
TODO: There a quite a few things you'll need to do in this function:
- Define the variables U, b_2.
- Define the vector h as a constant and inititalize it with
zeros. See tf.zeros and tf.shape for information on how
to initialize this variable to be of the right shape.
https://www.tensorflow.org/api_docs/python/constant_op/constant_value_tensors#zeros
https://www.tensorflow.org/api_docs/python/array_ops/shapes_and_shaping#shape
- In a for loop, begin to unroll the RNN sequence. Collect
the predictions in a list.
- When unrolling the loop, from the second iteration
onwards, you will HAVE to call
tf.get_variable_scope().reuse_variables() so that you do
not create new variables in the RNN cell.
See https://www.tensorflow.org/versions/master/how_tos/variable_scope/
- Concatenate and reshape the predictions into a predictions
tensor.
Hint: You will find the function tf.pack (similar to np.asarray)
useful to assemble a list of tensors into a larger tensor.
https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#pack
Hint: You will find the function tf.transpose and the perms
argument useful to shuffle the indices of the tensor.
https://www.tensorflow.org/api_docs/python/array_ops/slicing_and_joining#transpose
Remember:
* Use the xavier initilization for matrices.
* Note that tf.nn.dropout takes the keep probability (1 - p_drop) as an argument.
The keep probability should be set to the value of self.dropout_placeholder
Returns:
pred: tf.Tensor of shape (batch_size, max_length, n_classes)
"""
x1, x2 = self.add_embedding()
dropout_rate = self.dropout_placeholder
# choose cell type
if self.config.cell == "rnn":
cell = RNNCell(self.config.embed_size, self.config.hidden_size)
elif self.config.cell == "gru":
cell = GRUCell(self.config.embed_size, self.config.hidden_size)
elif self.config.cell == "lstm":
cell = LSTMCell(self.config.embed_size, self.config.hidden_size)
else:
raise ValueError("Unsuppported cell type: " + self.config.cell)
# Initialize hidden states to zero vectors of shape (num_examples, hidden_size)
h1 = tf.zeros((tf.shape(x1)[0], self.config.hidden_size), tf.float32)
h2 = tf.zeros((tf.shape(x2)[0], self.config.hidden_size), tf.float32)
with tf.variable_scope("RNN1") as scope:
for time_step in range(self.helper.max_length):
if time_step != 0:
scope.reuse_variables()
o1_t, h1 = cell(x1[:, time_step, :], h1, scope)
with tf.variable_scope("RNN2") as scope:
for time_step in range(self.helper.max_length):
if time_step != 0:
scope.reuse_variables()
o2_t, h2 = cell(x2[:, time_step, :], h2, scope)
# h_drop1 = tf.nn.dropout(h1, dropout_rate)
# h_drop2 = tf.nn.dropout(h2, dropout_rate)
# use L2-regularization: sum of squares of all parameters
if self.config.distance_measure == "l2":
# perform logistic regression on l2-distance between h1 and h2
distance = norm(h1 - h2 + 0.000001)
logistic_a = tf.Variable(0.0, dtype=tf.float32, name="logistic_a")
logistic_b = tf.Variable(0.0, dtype=tf.float32, name="logistic_b")
self.regularization_term = tf.square(logistic_a) + tf.square(
logistic_b)
preds = tf.sigmoid(logistic_a * distance + logistic_b)
elif self.config.distance_measure == "cosine":
# perform logistic regression on cosine distance between h1 and h2
distance = cosine_distance(h1 + 0.000001, h2 + 0.000001)
logistic_a = tf.Variable(1.0, dtype=tf.float32, name="logistic_a")
logistic_b = tf.Variable(0.0, dtype=tf.float32, name="logistic_b")
self.regularization_term = tf.square(logistic_a) + tf.square(
logistic_b)
preds = tf.sigmoid(logistic_a * distance + logistic_b)
elif self.config.distance_measure == "custom_coef":
# perform logistic regression on the vector |h1-h2|,
# equivalent to logistic regression on the (scalar) weighted Manhattan distance between h1 and h2,
# ie. weighted sum of |h1-h2|
logistic_a = tf.get_variable(
"coef", [self.config.hidden_size], tf.float32,
tf.contrib.layers.xavier_initializer())
logistic_b = tf.Variable(0.0, dtype=tf.float32, name="logistic_b")
self.regularization_term = tf.reduce_sum(
tf.square(logistic_a)) + tf.square(logistic_b)
preds = tf.sigmoid(
tf.reduce_sum(logistic_a * tf.abs(h1 - h2), axis=1) +
logistic_b)
elif self.config.distance_measure == "concat":
# use softmax for prediction
U = tf.get_variable(
"U", (4 * self.config.hidden_size, self.config.n_classes),
tf.float32, tf.contrib.layers.xavier_initializer())
b = tf.get_variable("b", (self.config.n_classes, ), tf.float32,
tf.constant_initializer(0))
v = tf.nn.relu(tf.concat([h1, h2, tf.square(h1 - h2), h1 * h2], 1))
self.regularization_term = tf.reduce_sum(
tf.square(U)) + tf.reduce_sum(tf.square(b))
preds = tf.matmul(v, U) + b
elif self.config.distance_measure == "concat_steroids":
# use softmax for prediction
W1 = tf.get_variable(
"W1", (4 * self.config.hidden_size, self.config.hidden_size),
tf.float32, tf.contrib.layers.xavier_initializer())
b1 = tf.get_variable("b1", (self.config.hidden_size, ), tf.float32,
tf.constant_initializer(0))
W2 = tf.get_variable(
"W2", (self.config.hidden_size, self.config.n_classes),
tf.float32, tf.contrib.layers.xavier_initializer())
b2 = tf.get_variable("b2", (self.config.n_classes, ), tf.float32,
tf.constant_initializer(0))
v1 = tf.nn.relu(tf.concat(
[h1, h2, tf.square(h1 - h2), h1 * h2], 1))
v2 = tf.nn.relu(tf.matmul(v1, W1) + b1)
self.regularization_term = tf.reduce_sum(
tf.square(W1)) + tf.reduce_sum(tf.square(b1)) + tf.reduce_sum(
tf.square(W2)) + tf.reduce_sum(tf.square(b2))
preds = tf.matmul(v2, W2) + b2
else:
raise ValueError("Unsuppported distance type: " +
self.config.distance_measure)
return preds
def add_prediction_op(self):
"""Adds the unrolled RNN:
h_0 = 0
for t in 1 to T:
o_t, h_t = cell(x_t, h_{t-1})
o_drop_t = Dropout(o_t, dropout_rate)
y_t = o_drop_t U + b_2
Remember:
* Use the xavier initilization for matrices.
* Note that tf.nn.dropout takes the keep probability (1 - p_drop) as an argument.
The keep probability should be set to the value of self.dropout_placeholder
Returns:
pred: tf.Tensor of shape (batch_size, max_length, n_classes)
"""
x = self.add_embedding()
dropout_rate = self.dropout_placeholder
preds = [] # Predicted output at each timestep should go here!
if self.config.cell == "rnn":
cell = RNNCell(Config.n_features * Config.embed_size,
Config.hidden_size)
elif self.config.cell == "gru":
cell = GRUCell(Config.n_features * Config.embed_size,
Config.hidden_size)
else:
raise ValueError("Unsuppported cell type: " + self.config.cell)
U = tf.get_variable(
"U",
shape=[self.config.hidden_size, self.config.n_classes],
dtype=tf.float32,
initializer=xavier_weight_init())
b2 = tf.get_variable("b2",
shape=[
self.config.n_classes,
],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
h_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size],
dtype=tf.float32)
x = tf.transpose(x, [1, 0, 2])
with tf.variable_scope("RNN"):
for time_step in range(self.max_length):
if (time_step > 0): tf.get_variable_scope().reuse_variables()
o_t, h_t = cell(x[time_step], h_t)
o_drop_t = tf.nn.dropout(o_t, 1 - self.config.dropout)
y_t = tf.matmul(o_drop_t, U) + b2
preds.append(y_t)
preds = tf.transpose(tf.stack(preds), [1, 0, 2])
assert preds.get_shape().as_list() == [
None, self.max_length, self.config.n_classes
], "predictions are not of the right shape. Expected {}, got {}".format(
[None, self.max_length, self.config.n_classes],
preds.get_shape().as_list())
return preds