def encode(self, inputs, sequence_length, **kwargs):
print("\n--------------------------\ninputs to encode\n" +
str(inputs) + "\n\n")
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
# Concatenate outputs and states of the forward and backward RNNs
outputs_concat = tf.concat(outputs, 2)
print("\noutput state tensor\n" + str(outputs_concat) + "\n\n")
print("\nfinal state tensor\n" + str(states) + "\n\n")
return EncoderOutput(outputs=outputs_concat,
final_state=states,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(tf.random_uniform_initializer(
-self.params["init_scale"],
self.params["init_scale"]))
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cells_fw = _unpack_cell(cell_fw)
cells_bw = _unpack_cell(cell_bw)
result = rnn.stack_bidirectional_dynamic_rnn(
cells_fw=cells_fw,
cells_bw=cells_bw,
inputs=inputs,
dtype=tf.float32,
sequence_length=sequence_length,
**kwargs)
outputs_concat, _output_state_fw, _output_state_bw = result
final_state = (_output_state_fw, _output_state_bw)
return EncoderOutput(
outputs=outputs_concat,
final_state=final_state,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cells_fw = _unpack_cell(cell_fw)
cells_bw = _unpack_cell(cell_bw)
result = rnn.stack_bidirectional_dynamic_rnn(
cells_fw=cells_fw,
cells_bw=cells_bw,
inputs=inputs,
dtype=tf.float32,
sequence_length=sequence_length,
**kwargs)
outputs_concat, _output_state_fw, _output_state_bw = result
final_state = (_output_state_fw, _output_state_bw)
return EncoderOutput(outputs=outputs_concat,
final_state=final_state,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
self.params["rnn_cell"]["distributed"] = False
self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName(
0)
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName(
self.params["rnn_cell"]["num_layers"])
if self.params["rnn_cell"][
"device_name"] == training_utils.getDeviceName(0):
self.params["rnn_cell"][
"device_name"] = training_utils.getDeviceName(
1
) # to ensure the backward cell is working on aniother GPU
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
# Concatenate outputs and states of the forward and backward RNNs
outputs_concat = tf.concat(outputs, 2)
return EncoderOutput(outputs=outputs_concat,
final_state=states,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode_decode(self, source, source_len, decoder_input_fn, target_len):
encoder_cell = training_utils.get_rnn_cell(
cell_type=self.params["rnn_cell.type"],
num_units=self.params["rnn_cell.num_units"],
num_layers=self.params["rnn_cell.num_layers"],
dropout_input_keep_prob=self.params["rnn_cell.dropout_input_keep_prob"],
dropout_output_keep_prob=self.params[
"rnn_cell.dropout_output_keep_prob"])
encoder_fn = encoders.BidirectionalRNNEncoder(encoder_cell)
encoder_output = encoder_fn(source, source_len)
decoder_cell = encoder_cell
decoder_fn = decoders.AttentionDecoder(
cell=decoder_cell,
vocab_size=self.target_vocab_info.total_size,
attention_inputs=encoder_output.outputs,
attention_fn=decoders.AttentionLayer(self.params["attention.dim"]),
max_decode_length=self.params["target.max_seq_len"])
decoder_output, _, _ = decoder_fn(
input_fn=decoder_input_fn,
initial_state=decoder_cell.zero_state(
tf.shape(source_len)[0], dtype=tf.float32),
sequence_length=target_len)
return decoder_output
def test_full_class_path(self):
cell = training_utils.get_rnn_cell(
cell_class="tensorflow.contrib.rnn.BasicRNNCell",
cell_params={"num_units": 16},
num_layers=1)
self.assertIsInstance(cell, tf.contrib.rnn.BasicRNNCell)
self.assertEqual(cell.output_size, 16)
def test_dropout(self):
cell = training_utils.get_rnn_cell(cell_class="BasicLSTMCell",
cell_params={"num_units": 16},
num_layers=1,
dropout_input_keep_prob=0.5)
self.assertIsInstance(cell, tf.contrib.rnn.DropoutWrapper)
self.assertEqual(cell.output_size, 16)
def __init__(self, params, mode, name): GraphModule.__init__(self, name) Configurable.__init__(self, params, mode) self.params["rnn_cell"] = _toggle_dropout(self.params["rnn_cell"], mode) self.cell = training_utils.get_rnn_cell(**self.params["rnn_cell"]) # Not initialized yet self.initial_state = None self.helper = None
def __init__(self, params, mode, name): GraphModule.__init__(self, name) Configurable.__init__(self, params, mode) self.params["rnn_cell"] = _toggle_dropout(self.params["rnn_cell"], mode) self.cell = training_utils.get_rnn_cell(**self.params["rnn_cell"]) # Not initialized yet self.initial_state = None self.helper = None
def test_multi_layer(self):
cell = training_utils.get_rnn_cell(cell_spec={
"class": "BasicLSTMCell",
"num_units": 16
},
num_layers=2)
self.assertIsInstance(cell, rnn_cell.ExtendedMultiRNNCell)
self.assertEqual(cell.output_size, 16)
def test_single_layer(self):
cell = training_utils.get_rnn_cell(cell_spec={
"class": "BasicLSTMCell",
"num_units": 16
},
num_layers=1)
self.assertIsInstance(cell, tf.contrib.rnn.BasicLSTMCell)
self.assertEqual(cell.output_size, 16)
def test_dropout(self):
cell = training_utils.get_rnn_cell(
cell_class="BasicLSTMCell",
cell_params={"num_units": 16},
num_layers=1,
dropout_input_keep_prob=0.5)
self.assertIsInstance(cell, tf.contrib.rnn.DropoutWrapper)
self.assertEqual(cell.output_size, 16)
def encode(self, inputs, sequence_length, **kwargs):
CONTEXT_SIZE = 10
inputs_c = tf.slice(inputs, [0, 0, 0], [-1, CONTEXT_SIZE, -1])
inputs_p = tf.slice(inputs, [0, CONTEXT_SIZE, 0], [-1, -1, -1])
with tf.variable_scope("encoder_c"):
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
cell_fw_c = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw_c = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs_c, states_c = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw_c,
cell_bw=cell_bw_c,
inputs=inputs_c,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
with tf.variable_scope("encoder_p"):
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
cell_fw_p = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw_p = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs_p, states_p = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw_p,
cell_bw=cell_bw_p,
inputs=inputs_p,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
# Concatenate outputs and states of the forward and backward RNNs
outputs_concat_c = tf.concat(outputs_c, 2)
outputs_concat_p = tf.concat(outputs_p, 2)
final_output = tf.concat([outputs_concat_c, outputs_concat_p], 1)
final_states = states_c + states_p
return EncoderOutput(outputs=final_output,
final_state=final_states,
attention_values=final_output,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
# Concatenate outputs and states of the forward and backward RNNs
outputs_concat = tf.concat(outputs, 2)
return EncoderOutput(
outputs=outputs_concat,
final_state=states,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def test_extra_args(self):
# Invalid args should raise a ValueError
with self.assertRaises(ValueError):
training_utils.get_rnn_cell(
cell_class="LSTMCell",
cell_params={"num_units": 16,
"use_peepholesERROR": True},
num_layers=1)
cell = training_utils.get_rnn_cell(
cell_class="LSTMCell",
cell_params={"num_units": 8,
"use_peepholes": True,
"forget_bias": 0.5},
num_layers=1)
self.assertIsInstance(cell, tf.contrib.rnn.LSTMCell)
#pylint: disable=E1101,W0212
self.assertEqual(cell._use_peepholes, True)
self.assertEqual(cell._forget_bias, 0.5)
self.assertEqual(cell.output_size, 8)
def __init__(self, params, mode, name):
GraphModule.__init__(self, name)
Configurable.__init__(self, params, mode)
self.params["rnn_cell"] = _toggle_dropout(
self.params["rnn_cell"],
["dropout_input_keep_prob", "dropout_output_keep_prob"], mode)
self.params["dropout_decoder"] = _toggle_dropout(
self.params["dropout_decoder"], ["dropout_deep_output_layer"],
mode)
with tf.variable_scope("rec1"):
self.cell_rec1 = training_utils.get_rnn_cell(
**self.params["rnn_cell"])
with tf.variable_scope("rec2"):
self.cell_rec2 = training_utils.get_rnn_cell(
**self.params["rnn_cell"])
# Not initialized yet
self.initial_state = None
self.helper = None
def encode(self, inputs, sequence_length, **kwargs):
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cells_fw = _unpack_cell(cell_fw)
cells_bw = _unpack_cell(cell_bw)
result = rnn.stack_bidirectional_dynamic_rnn(
cells_fw=cells_fw,
cells_bw=cells_bw,
inputs=inputs,
dtype=tf.float32,
sequence_length=sequence_length,
**kwargs)
outputs_concat, _output_state_fw, _output_state_bw = result
final_state = (_output_state_fw, _output_state_bw)
return EncoderOutput(
outputs=outputs_concat,
final_state=final_state,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
query_inputs, retrieved_inputs = tf.split(inputs, 2)
query_cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
query_cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
query_outputs, query_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=query_cell_fw,
cell_bw=query_cell_bw,
inputs=query_inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
retrieved_cell_fw = training_utils.get_rnn_cell(
**self.params["rnn_cell"])
retrieved_cell_bw = training_utils.get_rnn_cell(
**self.params["rnn_cell"])
retrieved_outputs, retrieved_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=retrieved_cell_fw,
cell_bw=retrieved_cell_bw,
inputs=retrieved_inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
outputs = tf.concat([query_outputs, retrieved_outputs], 0)
# Concatenate outputs and states of the forward and backward RNNs
outputs_concat = tf.concat(outputs, 2)
states = tf.concat([query_states, retrieved_states], 0)
return EncoderOutput(outputs=outputs_concat,
final_state=states,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
cell = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, state = tf.nn.dynamic_rnn(
cell=cell,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
return EncoderOutput(
outputs=outputs,
final_state=state,
attention_values=outputs,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(tf.random_uniform_initializer(
-self.params["init_scale"],
self.params["init_scale"]))
embedding_size = inputs.get_shape().as_list()[-1] # TODO: Different size for words and context
self.params["rnn_cell"]["cell_params"]["num_units"] = embedding_size + self.positional_embedding_size
inner_cell = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell = AttentionRNNCell(inner_cell,
embedding_size,
self.positional_embedding_size,
self.attention_num_layers,
self.attention_num_units)
positional_embeddings_var = tf.get_variable("positional_embeddings",
[self.max_sequence_length, self.positional_embedding_size],
dtype=tf.float32) # TODO: Make dtype configurable
position_sequence = tf.range(tf.shape(inputs)[1])
positional_embeddings = tf.nn.embedding_lookup(positional_embeddings_var, position_sequence)
positional_embeddings = tf.expand_dims(positional_embeddings, axis=0)
positional_embeddings_for_batch = tf.tile(positional_embeddings, [tf.shape(inputs)[0], 1, 1])
initial_state_0 = tf.zeros([tf.shape(inputs)[0], inner_cell.state_size])
initial_state_1 = tf.concat([inputs, positional_embeddings_for_batch], axis=2)
initial_state_2 = tf.concat([inputs, positional_embeddings_for_batch], axis=2)
initial_state = (initial_state_0, initial_state_1, initial_state_2)
outputs, state = tf.nn.dynamic_rnn(
cell=cell,
inputs=tf.zeros([tf.shape(inputs)[0], tf.shape(inputs)[1] * 1, 1], tf.float32),
# Todo : Make this * 1 configurable
initial_state=initial_state,
sequence_length=sequence_length * 1, # Todo : Make this 1 configurable
dtype=tf.float32,
**kwargs)
return EncoderOutput(
outputs=state[2],
final_state=state[0],
attention_values=state[2],
attention_values_length=tf.ones([tf.shape(inputs)[0]], dtype=tf.int32) * tf.shape(inputs)[1])
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
self.params["rnn_cell"]["distributed"] = False
self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName(
0)
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName(
self.params["rnn_cell"]["num_layers"])
if self.params["rnn_cell"][
"device_name"] == training_utils.getDeviceName(0):
self.params["rnn_cell"][
"device_name"] = training_utils.getDeviceName(
1
) # to ensure the backward cell is working on aniother GPU
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cells_fw = _unpack_cell(cell_fw)
cells_bw = _unpack_cell(cell_bw)
result = rnn.stack_bidirectional_dynamic_rnn(
cells_fw=cells_fw,
cells_bw=cells_bw,
inputs=inputs,
dtype=tf.float32,
sequence_length=sequence_length,
**kwargs)
outputs_concat, _output_state_fw, _output_state_bw = result
final_state = (_output_state_fw, _output_state_bw)
return EncoderOutput(outputs=outputs_concat,
final_state=final_state,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(tf.random_uniform_initializer(
-self.params["init_scale"],
self.params["init_scale"]))
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
# Concatenate outputs and states of the forward and backward RNNs
outputs_concat = tf.concat(outputs, 2)
return EncoderOutput(
outputs=outputs_concat,
final_state=states,
attention_values=outputs_concat,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, source_images, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(tf.contrib.layers.xavier_initializer())
cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
# Concatenate outputs and states of the forward and backward RNNs
outputs_concat = tf.concat(outputs, 2)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
moving_average_decay = self.params["resnet"][
"moving_average_decay"]
logits, end_points = _resnet_v1(
source_images,
states,
num_classes=None,
global_pool=True,
spatial_squeeze=False,
is_training=self.is_training,
moving_average_decay=moving_average_decay)
logits = tf.reshape(logits, [-1, 2048])
return EncoderOutput(outputs=outputs_concat,
final_state=states,
attention_values=outputs_concat,
attention_values_length=sequence_length,
image_features=logits)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
cell = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, state = tf.nn.dynamic_rnn(cell=cell,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
return EncoderOutput(outputs=outputs,
final_state=state,
attention_values=outputs,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
scope = tf.get_variable_scope()
scope.set_initializer(tf.random_uniform_initializer(
-self.params["init_scale"],
self.params["init_scale"]))
cell = training_utils.get_rnn_cell(**self.params["rnn_cell"])
outputs, state = tf.nn.dynamic_rnn(
cell=cell,
inputs=inputs,
sequence_length=sequence_length,
dtype=tf.float32,
**kwargs)
return EncoderOutput(
outputs=outputs,
final_state=state,
attention_values=outputs,
attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs):
num_layers_2 = self.params['rnn_cell_uni']['num_layers']
scope = tf.get_variable_scope()
scope.set_initializer(
tf.random_uniform_initializer(-self.params["init_scale"],
self.params["init_scale"]))
bi_encoder_output = self.bi_encoder.encode(inputs, sequence_length,
**kwargs)
uni_cell = training_utils.get_rnn_cell(**self.params["rnn_cell_uni"])
encoder_outputs, encoder_state = tf.nn.dynamic_rnn(
uni_cell,
bi_encoder_output.outputs,
dtype=tf.float32,
sequence_length=sequence_length)
#time_major=self.time_major TODO needs to be checked
encoder_state = (bi_encoder_output.final_state[1], ) + (
(encoder_state, ) if num_layers_2 == 1 else encoder_state)
return EncoderOutput(outputs=encoder_outputs,
final_state=encoder_state,
attention_values=encoder_outputs,
attention_values_length=sequence_length)
def test_single_layer(self):
cell = training_utils.get_rnn_cell(
cell_class="BasicLSTMCell", cell_params={"num_units": 16}, num_layers=1)
self.assertIsInstance(cell, tf.contrib.rnn.BasicLSTMCell)
self.assertEqual(cell.output_size, 16)
def test_multi_layer(self):
cell = training_utils.get_rnn_cell(
cell_class="BasicLSTMCell", cell_params={"num_units": 16}, num_layers=2)
self.assertIsInstance(cell, rnn_cell.ExtendedMultiRNNCell)
self.assertEqual(cell.output_size, 16)
