def encode(self, inputs, sequence_length, **kwargs): print("\n--------------------------\ninputs to encode\n" + str(inputs) + "\n\n") scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, states = tf.nn.bidirectional_dynamic_rnn( cell_fw=cell_fw, cell_bw=cell_bw, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) # Concatenate outputs and states of the forward and backward RNNs outputs_concat = tf.concat(outputs, 2) print("\noutput state tensor\n" + str(outputs_concat) + "\n\n") print("\nfinal state tensor\n" + str(states) + "\n\n") return EncoderOutput(outputs=outputs_concat, final_state=states, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer(tf.random_uniform_initializer( -self.params["init_scale"], self.params["init_scale"])) cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cells_fw = _unpack_cell(cell_fw) cells_bw = _unpack_cell(cell_bw) result = rnn.stack_bidirectional_dynamic_rnn( cells_fw=cells_fw, cells_bw=cells_bw, inputs=inputs, dtype=tf.float32, sequence_length=sequence_length, **kwargs) outputs_concat, _output_state_fw, _output_state_bw = result final_state = (_output_state_fw, _output_state_bw) return EncoderOutput( outputs=outputs_concat, final_state=final_state, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cells_fw = _unpack_cell(cell_fw) cells_bw = _unpack_cell(cell_bw) result = rnn.stack_bidirectional_dynamic_rnn( cells_fw=cells_fw, cells_bw=cells_bw, inputs=inputs, dtype=tf.float32, sequence_length=sequence_length, **kwargs) outputs_concat, _output_state_fw, _output_state_bw = result final_state = (_output_state_fw, _output_state_bw) return EncoderOutput(outputs=outputs_concat, final_state=final_state, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) self.params["rnn_cell"]["distributed"] = False self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName( 0) cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName( self.params["rnn_cell"]["num_layers"]) if self.params["rnn_cell"][ "device_name"] == training_utils.getDeviceName(0): self.params["rnn_cell"][ "device_name"] = training_utils.getDeviceName( 1 ) # to ensure the backward cell is working on aniother GPU cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, states = tf.nn.bidirectional_dynamic_rnn( cell_fw=cell_fw, cell_bw=cell_bw, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) # Concatenate outputs and states of the forward and backward RNNs outputs_concat = tf.concat(outputs, 2) return EncoderOutput(outputs=outputs_concat, final_state=states, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode_decode(self, source, source_len, decoder_input_fn, target_len): encoder_cell = training_utils.get_rnn_cell( cell_type=self.params["rnn_cell.type"], num_units=self.params["rnn_cell.num_units"], num_layers=self.params["rnn_cell.num_layers"], dropout_input_keep_prob=self.params["rnn_cell.dropout_input_keep_prob"], dropout_output_keep_prob=self.params[ "rnn_cell.dropout_output_keep_prob"]) encoder_fn = encoders.BidirectionalRNNEncoder(encoder_cell) encoder_output = encoder_fn(source, source_len) decoder_cell = encoder_cell decoder_fn = decoders.AttentionDecoder( cell=decoder_cell, vocab_size=self.target_vocab_info.total_size, attention_inputs=encoder_output.outputs, attention_fn=decoders.AttentionLayer(self.params["attention.dim"]), max_decode_length=self.params["target.max_seq_len"]) decoder_output, _, _ = decoder_fn( input_fn=decoder_input_fn, initial_state=decoder_cell.zero_state( tf.shape(source_len)[0], dtype=tf.float32), sequence_length=target_len) return decoder_output
def test_full_class_path(self): cell = training_utils.get_rnn_cell( cell_class="tensorflow.contrib.rnn.BasicRNNCell", cell_params={"num_units": 16}, num_layers=1) self.assertIsInstance(cell, tf.contrib.rnn.BasicRNNCell) self.assertEqual(cell.output_size, 16)
def test_dropout(self): cell = training_utils.get_rnn_cell(cell_class="BasicLSTMCell", cell_params={"num_units": 16}, num_layers=1, dropout_input_keep_prob=0.5) self.assertIsInstance(cell, tf.contrib.rnn.DropoutWrapper) self.assertEqual(cell.output_size, 16)
def __init__(self, params, mode, name): GraphModule.__init__(self, name) Configurable.__init__(self, params, mode) self.params["rnn_cell"] = _toggle_dropout(self.params["rnn_cell"], mode) self.cell = training_utils.get_rnn_cell(**self.params["rnn_cell"]) # Not initialized yet self.initial_state = None self.helper = None
def test_multi_layer(self): cell = training_utils.get_rnn_cell(cell_spec={ "class": "BasicLSTMCell", "num_units": 16 }, num_layers=2) self.assertIsInstance(cell, rnn_cell.ExtendedMultiRNNCell) self.assertEqual(cell.output_size, 16)
def test_single_layer(self): cell = training_utils.get_rnn_cell(cell_spec={ "class": "BasicLSTMCell", "num_units": 16 }, num_layers=1) self.assertIsInstance(cell, tf.contrib.rnn.BasicLSTMCell) self.assertEqual(cell.output_size, 16)
def test_dropout(self): cell = training_utils.get_rnn_cell( cell_class="BasicLSTMCell", cell_params={"num_units": 16}, num_layers=1, dropout_input_keep_prob=0.5) self.assertIsInstance(cell, tf.contrib.rnn.DropoutWrapper) self.assertEqual(cell.output_size, 16)
def encode(self, inputs, sequence_length, **kwargs): CONTEXT_SIZE = 10 inputs_c = tf.slice(inputs, [0, 0, 0], [-1, CONTEXT_SIZE, -1]) inputs_p = tf.slice(inputs, [0, CONTEXT_SIZE, 0], [-1, -1, -1]) with tf.variable_scope("encoder_c"): scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) cell_fw_c = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw_c = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs_c, states_c = tf.nn.bidirectional_dynamic_rnn( cell_fw=cell_fw_c, cell_bw=cell_bw_c, inputs=inputs_c, sequence_length=sequence_length, dtype=tf.float32, **kwargs) with tf.variable_scope("encoder_p"): scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) cell_fw_p = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw_p = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs_p, states_p = tf.nn.bidirectional_dynamic_rnn( cell_fw=cell_fw_p, cell_bw=cell_bw_p, inputs=inputs_p, sequence_length=sequence_length, dtype=tf.float32, **kwargs) # Concatenate outputs and states of the forward and backward RNNs outputs_concat_c = tf.concat(outputs_c, 2) outputs_concat_p = tf.concat(outputs_p, 2) final_output = tf.concat([outputs_concat_c, outputs_concat_p], 1) final_states = states_c + states_p return EncoderOutput(outputs=final_output, final_state=final_states, attention_values=final_output, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, states = tf.nn.bidirectional_dynamic_rnn( cell_fw=cell_fw, cell_bw=cell_bw, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) # Concatenate outputs and states of the forward and backward RNNs outputs_concat = tf.concat(outputs, 2) return EncoderOutput( outputs=outputs_concat, final_state=states, attention_values=outputs_concat, attention_values_length=sequence_length)
def test_extra_args(self): # Invalid args should raise a ValueError with self.assertRaises(ValueError): training_utils.get_rnn_cell( cell_class="LSTMCell", cell_params={"num_units": 16, "use_peepholesERROR": True}, num_layers=1) cell = training_utils.get_rnn_cell( cell_class="LSTMCell", cell_params={"num_units": 8, "use_peepholes": True, "forget_bias": 0.5}, num_layers=1) self.assertIsInstance(cell, tf.contrib.rnn.LSTMCell) #pylint: disable=E1101,W0212 self.assertEqual(cell._use_peepholes, True) self.assertEqual(cell._forget_bias, 0.5) self.assertEqual(cell.output_size, 8)
def __init__(self, params, mode, name): GraphModule.__init__(self, name) Configurable.__init__(self, params, mode) self.params["rnn_cell"] = _toggle_dropout( self.params["rnn_cell"], ["dropout_input_keep_prob", "dropout_output_keep_prob"], mode) self.params["dropout_decoder"] = _toggle_dropout( self.params["dropout_decoder"], ["dropout_deep_output_layer"], mode) with tf.variable_scope("rec1"): self.cell_rec1 = training_utils.get_rnn_cell( **self.params["rnn_cell"]) with tf.variable_scope("rec2"): self.cell_rec2 = training_utils.get_rnn_cell( **self.params["rnn_cell"]) # Not initialized yet self.initial_state = None self.helper = None
def encode(self, inputs, sequence_length, **kwargs): cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cells_fw = _unpack_cell(cell_fw) cells_bw = _unpack_cell(cell_bw) result = rnn.stack_bidirectional_dynamic_rnn( cells_fw=cells_fw, cells_bw=cells_bw, inputs=inputs, dtype=tf.float32, sequence_length=sequence_length, **kwargs) outputs_concat, _output_state_fw, _output_state_bw = result final_state = (_output_state_fw, _output_state_bw) return EncoderOutput( outputs=outputs_concat, final_state=final_state, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) query_inputs, retrieved_inputs = tf.split(inputs, 2) query_cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) query_cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) query_outputs, query_states = tf.nn.bidirectional_dynamic_rnn( cell_fw=query_cell_fw, cell_bw=query_cell_bw, inputs=query_inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) retrieved_cell_fw = training_utils.get_rnn_cell( **self.params["rnn_cell"]) retrieved_cell_bw = training_utils.get_rnn_cell( **self.params["rnn_cell"]) retrieved_outputs, retrieved_states = tf.nn.bidirectional_dynamic_rnn( cell_fw=retrieved_cell_fw, cell_bw=retrieved_cell_bw, inputs=retrieved_inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) outputs = tf.concat([query_outputs, retrieved_outputs], 0) # Concatenate outputs and states of the forward and backward RNNs outputs_concat = tf.concat(outputs, 2) states = tf.concat([query_states, retrieved_states], 0) return EncoderOutput(outputs=outputs_concat, final_state=states, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): cell = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, state = tf.nn.dynamic_rnn( cell=cell, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) return EncoderOutput( outputs=outputs, final_state=state, attention_values=outputs, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer(tf.random_uniform_initializer( -self.params["init_scale"], self.params["init_scale"])) embedding_size = inputs.get_shape().as_list()[-1] # TODO: Different size for words and context self.params["rnn_cell"]["cell_params"]["num_units"] = embedding_size + self.positional_embedding_size inner_cell = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell = AttentionRNNCell(inner_cell, embedding_size, self.positional_embedding_size, self.attention_num_layers, self.attention_num_units) positional_embeddings_var = tf.get_variable("positional_embeddings", [self.max_sequence_length, self.positional_embedding_size], dtype=tf.float32) # TODO: Make dtype configurable position_sequence = tf.range(tf.shape(inputs)[1]) positional_embeddings = tf.nn.embedding_lookup(positional_embeddings_var, position_sequence) positional_embeddings = tf.expand_dims(positional_embeddings, axis=0) positional_embeddings_for_batch = tf.tile(positional_embeddings, [tf.shape(inputs)[0], 1, 1]) initial_state_0 = tf.zeros([tf.shape(inputs)[0], inner_cell.state_size]) initial_state_1 = tf.concat([inputs, positional_embeddings_for_batch], axis=2) initial_state_2 = tf.concat([inputs, positional_embeddings_for_batch], axis=2) initial_state = (initial_state_0, initial_state_1, initial_state_2) outputs, state = tf.nn.dynamic_rnn( cell=cell, inputs=tf.zeros([tf.shape(inputs)[0], tf.shape(inputs)[1] * 1, 1], tf.float32), # Todo : Make this * 1 configurable initial_state=initial_state, sequence_length=sequence_length * 1, # Todo : Make this 1 configurable dtype=tf.float32, **kwargs) return EncoderOutput( outputs=state[2], final_state=state[0], attention_values=state[2], attention_values_length=tf.ones([tf.shape(inputs)[0]], dtype=tf.int32) * tf.shape(inputs)[1])
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) self.params["rnn_cell"]["distributed"] = False self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName( 0) cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) self.params["rnn_cell"]["device_name"] = training_utils.getDeviceName( self.params["rnn_cell"]["num_layers"]) if self.params["rnn_cell"][ "device_name"] == training_utils.getDeviceName(0): self.params["rnn_cell"][ "device_name"] = training_utils.getDeviceName( 1 ) # to ensure the backward cell is working on aniother GPU cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cells_fw = _unpack_cell(cell_fw) cells_bw = _unpack_cell(cell_bw) result = rnn.stack_bidirectional_dynamic_rnn( cells_fw=cells_fw, cells_bw=cells_bw, inputs=inputs, dtype=tf.float32, sequence_length=sequence_length, **kwargs) outputs_concat, _output_state_fw, _output_state_bw = result final_state = (_output_state_fw, _output_state_bw) return EncoderOutput(outputs=outputs_concat, final_state=final_state, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer(tf.random_uniform_initializer( -self.params["init_scale"], self.params["init_scale"])) cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, states = tf.nn.bidirectional_dynamic_rnn( cell_fw=cell_fw, cell_bw=cell_bw, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) # Concatenate outputs and states of the forward and backward RNNs outputs_concat = tf.concat(outputs, 2) return EncoderOutput( outputs=outputs_concat, final_state=states, attention_values=outputs_concat, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, source_images, **kwargs): scope = tf.get_variable_scope() scope.set_initializer(tf.contrib.layers.xavier_initializer()) cell_fw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) cell_bw = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, states = tf.nn.bidirectional_dynamic_rnn( cell_fw=cell_fw, cell_bw=cell_bw, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) # Concatenate outputs and states of the forward and backward RNNs outputs_concat = tf.concat(outputs, 2) with slim.arg_scope(resnet_utils.resnet_arg_scope()): moving_average_decay = self.params["resnet"][ "moving_average_decay"] logits, end_points = _resnet_v1( source_images, states, num_classes=None, global_pool=True, spatial_squeeze=False, is_training=self.is_training, moving_average_decay=moving_average_decay) logits = tf.reshape(logits, [-1, 2048]) return EncoderOutput(outputs=outputs_concat, final_state=states, attention_values=outputs_concat, attention_values_length=sequence_length, image_features=logits)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) cell = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, state = tf.nn.dynamic_rnn(cell=cell, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) return EncoderOutput(outputs=outputs, final_state=state, attention_values=outputs, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): scope = tf.get_variable_scope() scope.set_initializer(tf.random_uniform_initializer( -self.params["init_scale"], self.params["init_scale"])) cell = training_utils.get_rnn_cell(**self.params["rnn_cell"]) outputs, state = tf.nn.dynamic_rnn( cell=cell, inputs=inputs, sequence_length=sequence_length, dtype=tf.float32, **kwargs) return EncoderOutput( outputs=outputs, final_state=state, attention_values=outputs, attention_values_length=sequence_length)
def encode(self, inputs, sequence_length, **kwargs): num_layers_2 = self.params['rnn_cell_uni']['num_layers'] scope = tf.get_variable_scope() scope.set_initializer( tf.random_uniform_initializer(-self.params["init_scale"], self.params["init_scale"])) bi_encoder_output = self.bi_encoder.encode(inputs, sequence_length, **kwargs) uni_cell = training_utils.get_rnn_cell(**self.params["rnn_cell_uni"]) encoder_outputs, encoder_state = tf.nn.dynamic_rnn( uni_cell, bi_encoder_output.outputs, dtype=tf.float32, sequence_length=sequence_length) #time_major=self.time_major TODO needs to be checked encoder_state = (bi_encoder_output.final_state[1], ) + ( (encoder_state, ) if num_layers_2 == 1 else encoder_state) return EncoderOutput(outputs=encoder_outputs, final_state=encoder_state, attention_values=encoder_outputs, attention_values_length=sequence_length)
def test_single_layer(self): cell = training_utils.get_rnn_cell( cell_class="BasicLSTMCell", cell_params={"num_units": 16}, num_layers=1) self.assertIsInstance(cell, tf.contrib.rnn.BasicLSTMCell) self.assertEqual(cell.output_size, 16)
def test_multi_layer(self): cell = training_utils.get_rnn_cell( cell_class="BasicLSTMCell", cell_params={"num_units": 16}, num_layers=2) self.assertIsInstance(cell, rnn_cell.ExtendedMultiRNNCell) self.assertEqual(cell.output_size, 16)