def rnn_cells(self) -> RNNCellTuple:
"""Return the graph template to for creating RNN memory cells."""
if self.use_noisy_activations:
return (NoisyGRUCell(self.rnn_size, self.train_mode),
NoisyGRUCell(self.rnn_size, self.train_mode))
return (OrthoGRUCell(self.rnn_size), OrthoGRUCell(self.rnn_size))
def _get_rnn_cell(self):
"""Return the RNN cell for the encoder"""
# pylint: disable=redefined-variable-type
if self.use_noisy_activations:
cell = NoisyGRUCell(self.rnn_size, self.is_training)
else:
cell = tf.nn.rnn_cell.GRUCell(self.rnn_size)
return cell
def _get_rnn_cell(self):
"""Return the RNN cell for the encoder"""
if self.use_noisy_activations:
cell = NoisyGRUCell(self.rnn_size, self.is_training)
else:
cell = tf.nn.rnn_cell.GRUCell(self.rnn_size)
if self.use_pervasive_dropout:
shape = tf.concat(0, [tf.shape(self.inputs[0]), [rnn_size]])
## TODO shape needs recomputing
dropout_mask = tf.floor(
tf.random_uniform(shape, 0.0, 1.0) + self.dropout_placeholder)
scale = tf.inv(self.dropout_placeholder)
cell = PervasiveDropoutWrapper(cell, dropout_mask, scale)
return cell
def __init__(self, max_input_len, vocabulary, data_id, embedding_size,
rnn_size, name, dropout_keep_p=0.5,
use_noisy_activations=False, use_pervasive_dropout=False,
attention_type=None, attention_fertility=3,
parent_encoder=None):
self.name = name
self.max_input_len = max_input_len
assert_type(self, 'vocabulary', vocabulary, Vocabulary)
self.vocabulary = vocabulary
self.data_id = data_id
self.embedding_size = embedding_size
self.rnn_size = rnn_size
self.max_input_len = max_input_len
self.dropout_keep_p = dropout_keep_p
self.use_noisy_activations = use_noisy_activations
self.use_pervasive_dropout = use_pervasive_dropout
self.attention_type = attention_type
self.attention_fertility = attention_fertility
assert_type(self, 'parent_encoder', parent_encoder, SentenceEncoder,
can_be_none=True)
self.parent_encoder = parent_encoder
log("Initializing sentence encoder, name: \"{}\"".format(name))
with tf.variable_scope(name):
self.dropout_placeholder = tf.placeholder(tf.float32,
name="dropout")
self.is_training = tf.placeholder(tf.bool, name="is_training")
self.inputs = tf.placeholder(tf.int32, shape=[None, max_input_len + 2], name="encoder_inputs")
self.sentence_lengths = tf.placeholder(tf.int64, shape=[None], name="encoder_input_lengths")
# TODO remove this placeholder and refactor attention to work
# with sentence_lengths
self.weight_tensor = tf.placeholder(tf.float32, name="encoder_padding_weights")
if parent_encoder:
self.word_embeddings = parent_encoder.word_embeddings
else:
# NOTE the same note as in decoder when initializing embeddings
self.word_embeddings = tf.get_variable(
"word_embeddings", shape=[len(vocabulary), embedding_size],
initializer=tf.random_normal_initializer(stddev=0.01))
embedded_inputs = tf.nn.embedding_lookup(self.word_embeddings, self.inputs)
dropped_embedded_inputs = tf.nn.dropout(embedded_inputs, self.dropout_placeholder)
if parent_encoder:
self.forward_gru = parent_encoder.forward_gru
self.backward_gru = parent_encoder.backward_gru
else:
if use_noisy_activations:
self.forward_gru = NoisyGRUCell(rnn_size, self.is_training)
self.backward_gru = NoisyGRUCell(rnn_size, self.is_training)
else:
### this is used most of the time...
self.forward_gru = OrthoGRUCell(rnn_size)
self.backward_gru = OrthoGRUCell(rnn_size)
if use_pervasive_dropout:
# create dropout mask (shape batch x rnn_size)
# floor (random uniform + dropout_keep)
shape = tf.concat(0, [tf.shape(self.inputs[0]), [rnn_size]])
forward_dropout_mask = tf.floor(
tf.random_uniform(shape, 0.0, 1.0) + self.dropout_placeholder)
backward_dropout_mask = tf.floor(
tf.random_uniform(shape, 0.0, 1.0) + self.dropout_placeholder)
scale = tf.inv(self.dropout_placeholder)
self.forward_gru = PervasiveDropoutWrapper(
self.forward_gru, forward_dropout_mask, scale)
self.backward_gru = PervasiveDropoutWrapper(
self.backward_gru, backward_dropout_mask, scale)
outputs_bidi_t, encoded_t = tf.nn.bidirectional_dynamic_rnn(
self.forward_gru, self.backward_gru,
dropped_embedded_inputs, self.sentence_lengths,
dtype=tf.float32)
self.attention_tensor = tf.concat(2, outputs_bidi_t)
self.encoded = tf.concat(1, encoded_t)
self.attention_object = attention_type(
self.attention_tensor, scope="attention_{}".format(name),
dropout_placeholder=self.dropout_placeholder,
input_weights=self.weight_tensor,
max_fertility=attention_fertility) if attention_type else None
log("Sentence encoder initialized")
def get_rnn_cell():
if use_noisy_activations:
return NoisyGRUCell(rnn_size, training=self.is_training)
else:
return tf.nn.rnn_cell.GRUCell(rnn_size)
def __init__(self, max_input_len, vocabulary, data_id, embedding_size,
rnn_size, name, dropout_keep_p=0.5,
use_noisy_activations=False, use_pervasive_dropout=False,
attention_type=None, attention_fertility=3,
parent_encoder=None):
self.name = name
self.max_input_len = max_input_len
assert_type(self, 'vocabulary', vocabulary, Vocabulary)
self.vocabulary = vocabulary
self.data_id = data_id
self.embedding_size = embedding_size
self.rnn_size = rnn_size
self.max_input_len = max_input_len
self.dropout_keep_p = dropout_keep_p
self.use_noisy_activations = use_noisy_activations
self.use_pervasive_dropout = use_pervasive_dropout
self.attention_type = attention_type
self.attention_fertility = attention_fertility
assert_type(self, 'parent_encoder', parent_encoder, SentenceEncoder,
can_be_none=True)
self.parent_encoder = parent_encoder
log("Initializing sentence encoder, name: \"{}\"".format(name))
with tf.variable_scope(name):
self.dropout_placeholder = tf.placeholder(tf.float32,
name="dropout")
self.is_training = tf.placeholder(tf.bool, name="is_training")
self.inputs = [tf.placeholder(tf.int32, shape=[None],
name="input_{}".format(i))
for i in range(max_input_len + 2)]
self.weight_ins = [tf.placeholder(tf.float32, shape=[None],
name="input_{}".format(i))
for i in range(max_input_len + 2)]
self.weight_tensor = tf.concat(1, [tf.expand_dims(w, 1)
for w in self.weight_ins])
self.sentence_lengths = tf.to_int64(sum(self.weight_ins))
if parent_encoder:
self.word_embeddings = parent_encoder.word_embeddings
else:
self.word_embeddings = tf.Variable(tf.random_uniform(
[len(vocabulary), embedding_size], -1.0, 1.0))
embedded_inputs = [tf.nn.embedding_lookup(self.word_embeddings, i)
for i in self.inputs]
dropped_embedded_inputs = [
tf.nn.dropout(i, self.dropout_placeholder)
for i in embedded_inputs]
if parent_encoder:
self.forward_gru = parent_encoder.forward_gru
self.backward_gru = parent_encoder.backward_gru
else:
if use_noisy_activations:
self.forward_gru = NoisyGRUCell(rnn_size, self.is_training)
self.backward_gru = NoisyGRUCell(rnn_size, self.is_training)
else:
self.forward_gru = tf.nn.rnn_cell.GRUCell(rnn_size)
self.backward_gru = tf.nn.rnn_cell.GRUCell(rnn_size)
if use_pervasive_dropout:
# create dropout mask (shape batch x rnn_size)
# floor (random uniform + dropout_keep)
shape = tf.concat(0, [tf.shape(self.inputs[0]), [rnn_size]])
forward_dropout_mask = tf.floor(
tf.random_uniform(shape, 0.0, 1.0) + self.dropout_placeholder)
backward_dropout_mask = tf.floor(
tf.random_uniform(shape, 0.0, 1.0) + self.dropout_placeholder)
scale = tf.inv(self.dropout_placeholder)
self.forward_gru = PervasiveDropoutWrapper(
self.forward_gru, forward_dropout_mask, scale)
self.backward_gru = PervasiveDropoutWrapper(
self.backward_gru, backward_dropout_mask, scale)
bidi_layer = BidirectionalRNNLayer(self.forward_gru,
self.backward_gru,
dropped_embedded_inputs,
self.sentence_lengths)
self.outputs_bidi = bidi_layer.outputs_bidi
self.encoded = bidi_layer.encoded
self.attention_tensor = tf.concat(1, [tf.expand_dims(o, 1)
for o in self.outputs_bidi])
self.attention_object = attention_type(
self.attention_tensor, scope="attention_{}".format(name),
dropout_placeholder=self.dropout_placeholder,
input_weights=self.weight_tensor,
max_fertility=attention_fertility) if attention_type else None
log("Sentence encoder initialized")