def setup_target_encoder(self): """ This sets up an encoder that works on target sentence and produce a single label in the end encoder has attentions Returns ------- """ if self.num_layers > 1: self.tgt_encoder_cell = rnn_cell.GRUCell(self.size, input_size=self.embedding[1]) self.attn_cell = GRUCellAttn(self.size, self.encoder_output, scope="EncoderAttnCell") out = self.decoder_inputs with vs.variable_scope("TgtEncoder"): inp = self.decoder_inputs for i in xrange(self.num_layers - 1): with vs.variable_scope("TgtEncoderCell%d" % i) as scope: out, state_output = rnn.dynamic_rnn(self.tgt_encoder_cell, self.dropout(inp), time_major=False, dtype=dtypes.float32, sequence_length=self.tgt_steps, scope=scope, initial_state=self.tgt_encoder_state_output[i]) inp = out self.tgt_encoder_state_output.append(state_output) with vs.variable_scope("TgtEncoderAttnCell") as scope: out, state_output = rnn.dynamic_rnn(self.attn_cell, self.dropout(inp), time_major=False, dtype=dtypes.float32, sequence_length=self.tgt_steps, scope=scope, initial_state=self.tgt_encoder_state_output[i + 1]) self.tgt_encoder_output = out self.tgt_encoder_state_output.append(state_output)
def _composition_function(self, inputs, length, init_state=None): if self._composition == "GRU": cell = GRUCell(self._size) return dynamic_rnn(cell, inputs, sequence_length=length, time_major=True, initial_state=init_state, dtype=tf.float32)[0] elif self._composition == "LSTM": cell = BasicLSTMCell(self._size) init_state = tf.concat(1, [tf.zeros_like(init_state, tf.float32), init_state]) if init_state else None outs = dynamic_rnn(cell, inputs, sequence_length=length, time_major=True, initial_state=init_state, dtype=tf.float32)[0] return outs elif self._composition == "BiGRU": cell = GRUCell(self._size // 2, self._size) init_state_fw, init_state_bw = tf.split(1, 2, init_state) if init_state else (None, None) with tf.variable_scope("forward"): fw_outs = dynamic_rnn(cell, inputs, sequence_length=length, time_major=True, initial_state=init_state_fw, dtype=tf.float32)[0] with tf.variable_scope("backward"): rev_inputs = tf.reverse_sequence(tf.pack(inputs), length, 0, 1) rev_inputs = [tf.reshape(x, [-1, self._size]) for x in tf.split(0, len(inputs), rev_inputs)] bw_outs = dynamic_rnn(cell, rev_inputs, sequence_length=length, time_major=True, initial_state=init_state_bw, dtype=tf.float32)[0] bw_outs = tf.reverse_sequence(tf.pack(bw_outs), length, 0, 1) bw_outs = [tf.reshape(x, [-1, self._size]) for x in tf.split(0, len(inputs), bw_outs)] return [tf.concat(1, [fw_out, bw_out]) for fw_out, bw_out in zip(fw_outs, bw_outs)] else: raise NotImplementedError("Other compositions not implemented yet.")
def setup_decoder(self): """ This sets up a decoder but we may need a double-encoder Returns ------- """ if self.num_layers > 1: self.decoder_cell = rnn_cell.GRUCell(self.size, input_size=self.embedding[1]) self.attn_cell = GRUCellAttn(self.size, self.encoder_output, scope="DecoderAttnCell") out = self.decoder_inputs with vs.variable_scope("Decoder"): inp = self.decoder_inputs for i in xrange(self.num_layers - 1): with vs.variable_scope("DecoderCell%d" % i) as scope: out, state_output = rnn.dynamic_rnn(self.decoder_cell, self.dropout(inp), time_major=False, dtype=dtypes.float32, sequence_length=self.tgt_steps, scope=scope, initial_state=self.decoder_state_input[i]) inp = out self.decoder_state_output.append(state_output) with vs.variable_scope("DecoderAttnCell") as scope: out, state_output = rnn.dynamic_rnn(self.attn_cell, self.dropout(inp), time_major=False, dtype=dtypes.float32, sequence_length=self.tgt_steps, scope=scope, initial_state=self.decoder_state_input[i + 1]) self.decoder_output = out self.decoder_state_output.append(state_output)
def testBatchSizeFromInput(self): cell = Plus1RNNCell() # With static batch size inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5)) # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) self.assertEqual(3, outputs.shape[0].value) self.assertEqual(3, state.shape[0].value) # - With initial_state outputs, state = rnn.dynamic_rnn( cell, inputs, initial_state=array_ops.placeholder(dtypes.float32, shape=(3, 5))) self.assertEqual(3, outputs.shape[0].value) self.assertEqual(3, state.shape[0].value) # Without static batch size inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5)) # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) self.assertEqual(None, outputs.shape[0].value) self.assertEqual(None, state.shape[0].value) # - With initial_state outputs, state = rnn.dynamic_rnn( cell, inputs, initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5))) self.assertEqual(None, outputs.shape[0].value) self.assertEqual(None, state.shape[0].value)
def sentence_embedding_rnn(_encoder_inputs, vocab_size, cell, embedding_size, mask=None, dtype=dtypes.float32, scope=None, reuse_scop=None): """ """ with variable_scope.variable_scope("embedding_rnn", reuse=reuse_scop): # encoder_cell = rnn_cell.EmbeddingWrapper( # cell, embedding_classes=vocab_size, # embedding_size=embedding_size) # Divde encoder_inputs by given input_mask if mask != None: encoder_inputs = [[] for _ in mask] _mask = 0 for num in range(len(_encoder_inputs)): encoder_inputs[_mask].append(_encoder_inputs[num]) if num == mask[_mask]: _mask += 1 else: encoder_inputs = [] encoder_inputs.append(_encoder_inputs) encoder_state = None encoder_states = [] for encoder_input in encoder_inputs: if encoder_state == []: _, encoder_state = rnn.dynamic_rnn(encoder_cell, encoder_input, dtype=dtype) else: _, encoder_state = rnn.dynamic_rnn(encoder_cell, encoder_input, encoder_state, dtype=dtype) encoder_states.append(encoder_state) return encoder_states
def testBlockGRUToGRUCellMultiStep(self): with self.session(use_gpu=True, graph=ops.Graph()) as sess: batch_size = 2 cell_size = 3 input_size = 3 time_steps = 4 # Random initializers. seed = 1994 initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=seed) np.random.seed(seed) # Inputs concat_x = array_ops.placeholder( dtypes.float32, shape=(time_steps, batch_size, input_size)) h = array_ops.zeros([batch_size, cell_size]) # Values for the inputs. x_values = np.random.rand(time_steps, batch_size, input_size) h_value = np.random.rand(batch_size, cell_size) # Output from the block GRU cell implementation. with vs.variable_scope("block", initializer=initializer): cell = gru_ops.GRUBlockCell(cell_size) outputs_dynamic, state_dynamic = rnn.dynamic_rnn( cell, inputs=concat_x, initial_state=h, time_major=True, dtype=dtypes.float32) feeds = {concat_x: x_values, h: h_value} sess.run([variables.global_variables_initializer()]) block_res = sess.run([outputs_dynamic, state_dynamic], feeds) # Output from the basic GRU cell implementation. with vs.variable_scope("basic", initializer=initializer): cell = rnn_cell.GRUCell(cell_size) outputs_dynamic, state_dynamic = rnn.dynamic_rnn( cell, inputs=concat_x, initial_state=h, time_major=True, dtype=dtypes.float32) feeds = {concat_x: x_values, h: h_value} sess.run([variables.global_variables_initializer()]) basic_res = sess.run([outputs_dynamic, state_dynamic], feeds) # Check the lengths of the outputs_dynamic, and states. self.assertEqual(len(block_res), len(basic_res)) self.assertEqual(len(block_res[0]), len(basic_res[0])) self.assertEqual(len(block_res[1]), len(basic_res[1])) # Check the outputs_dynamic values. for block_output, basic_output in zip(block_res[0], basic_res[0]): self.assertAllClose(block_output, basic_output) # Check the state_dynamic value. self.assertAllClose(block_res[1], block_res[1])
def testInvalidSequenceLengthShape(self): cell = Plus1RNNCell() inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))] with self.assertRaisesRegexp(ValueError, "must be a vector"): rnn.dynamic_rnn( cell, array_ops.stack(inputs), dtype=dtypes.float32, sequence_length=[[4]])
def inference_gru_block_vs_gru_cell(batch_size, cell_size, input_size, time_steps, use_gpu=False, iters=30): """Benchmark inference speed between GRUBlockCell vs GRUCell.""" ops.reset_default_graph() with session.Session(graph=ops.Graph()) as sess: with benchmarking.device(use_gpu): # Random initializers. seed = 1994 initializer = init_ops.random_uniform_initializer(-1, 1, seed=seed) np.random.seed(seed) # Inputs concat_x = vs.get_variable("concat_x", [time_steps, batch_size, input_size]) h = vs.get_variable("h", [batch_size, cell_size]) # Output from the basic GRU cell implementation. with vs.variable_scope("basic", initializer=initializer): cell = rnn_cell.GRUCell(cell_size) outputs_dynamic, _ = rnn.dynamic_rnn( cell, inputs=concat_x, initial_state=h, time_major=True, dtype=dtypes.float32) sess.run([variables.global_variables_initializer()]) basic_time_inference = benchmarking.seconds_per_run( outputs_dynamic, sess, iters) # Output from the block GRU cell implementation. with vs.variable_scope("block", initializer=initializer): cell = gru_ops.GRUBlockCell(cell_size) outputs_dynamic, _ = rnn.dynamic_rnn( cell, inputs=concat_x, initial_state=h, time_major=True, dtype=dtypes.float32) sess.run([variables.global_variables_initializer()]) block_time_inference = benchmarking.seconds_per_run( outputs_dynamic, sess, iters) performance_inference = (basic_time_inference - block_time_inference ) * 100 / basic_time_inference print(",".join([ str(batch_size), str(cell_size), str(input_size), str(time_steps), str( use_gpu), str(basic_time_inference), str(block_time_inference), str( performance_inference) ])) return basic_time_inference, block_time_inference
def crf_decode(potentials, transition_params, sequence_length): """Decode the highest scoring sequence of tags in TensorFlow. This is a function for tensor. Args: potentials: A [batch_size, max_seq_len, num_tags] tensor of unary potentials. transition_params: A [num_tags, num_tags] matrix of binary potentials. sequence_length: A [batch_size] vector of true sequence lengths. Returns: decode_tags: A [batch_size, max_seq_len] matrix, with dtype `tf.int32`. Contains the highest scoring tag indicies. best_score: A [batch_size] vector, containing the score of `decode_tags`. """ # For simplicity, in shape comments, denote: # 'batch_size' by 'B', 'max_seq_len' by 'T' , 'num_tags' by 'O' (output). num_tags = potentials.get_shape()[2].value # Computes forward decoding. Get last score and backpointers. crf_fwd_cell = CrfDecodeForwardRnnCell(transition_params) initial_state = array_ops.slice(potentials, [0, 0, 0], [-1, 1, -1]) initial_state = array_ops.squeeze(initial_state, axis=[1]) # [B, O] inputs = array_ops.slice(potentials, [0, 1, 0], [-1, -1, -1]) # [B, T-1, O] backpointers, last_score = rnn.dynamic_rnn( crf_fwd_cell, inputs=inputs, sequence_length=sequence_length - 1, initial_state=initial_state, time_major=False, dtype=dtypes.int32) # [B, T - 1, O], [B, O] backpointers = gen_array_ops.reverse_sequence( backpointers, sequence_length - 1, seq_dim=1) # [B, T-1, O] # Computes backward decoding. Extract tag indices from backpointers. crf_bwd_cell = CrfDecodeBackwardRnnCell(num_tags) initial_state = math_ops.cast(math_ops.argmax(last_score, axis=1), dtype=dtypes.int32) # [B] initial_state = array_ops.expand_dims(initial_state, axis=-1) # [B, 1] decode_tags, _ = rnn.dynamic_rnn( crf_bwd_cell, inputs=backpointers, sequence_length=sequence_length - 1, initial_state=initial_state, time_major=False, dtype=dtypes.int32) # [B, T - 1, 1] decode_tags = array_ops.squeeze(decode_tags, axis=[2]) # [B, T - 1] decode_tags = array_ops.concat([initial_state, decode_tags], axis=1) # [B, T] decode_tags = gen_array_ops.reverse_sequence( decode_tags, sequence_length, seq_dim=1) # [B, T] best_score = math_ops.reduce_max(last_score, axis=1) # [B] return decode_tags, best_score
def testInvalidSequenceLengthShape(self): cell = Plus1RNNCell() if context.in_graph_mode(): inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))] else: inputs = [constant_op.constant(np.ones((3, 4)))] with self.assertRaisesRegexp(ValueError, "must be a vector"): rnn.dynamic_rnn( cell, array_ops.stack(inputs), dtype=dtypes.float32, sequence_length=[[4]])
def RNN(inputs, lens, name, reuse): print ("Building network " + name) # Define weights inputs = tf.gather(one_hots, inputs) weights = tf.Variable(tf.random_normal([__n_hidden, n_output]), name=name+"_weights") biases = tf.Variable(tf.random_normal([n_output]), name=name+"_biases") # Define a lstm cell with tensorflow enc_outputs, enc_states = rnn.dynamic_rnn( __cell_kind(__n_hidden), inputs, sequence_length=lens, dtype=tf.float32, scope=name, time_major=False) dec_outputs, dec_states = rnn.dynamic_rnn( __cell_kind(__n_hidden), enc_outputs, sequence_length=lens, dtype=tf.float32, scope=name, time_major=False) # Prepare data shape to match `rnn` function requirements # Current data input shape: (__batch_size, __n_steps, n_input) # Required shape: '__n_steps' tensors list of shape (__batch_size, n_input) '''dec_outputs, dec_states = rnn.rnn( __cell_kind(__n_hidden), tf.unpack(tf.transpose(inputs, [1, 0, 2])), sequence_length=lens, dtype=tf.float32, scope=name) outputs = tf.transpose(tf.pack(outputs), [1, 0, 2])''' print ("Done building network " + name) # Asserts are actually documentation: they can't be out of date assert dec_outputs.get_shape() == (__batch_size, __n_steps, __n_hidden) # Linear activation, using rnn output for each char # Reshaping here for a `batch` matrix multiply # It's faster than `batch_matmul` probably because it can guarantee a # static shape outputs = tf.reshape(dec_outputs, [__batch_size * __n_steps, __n_hidden]) finals = tf.matmul(outputs, weights) finals = tf.reshape(finals, [__batch_size, __n_steps, n_output]) + biases return finals[:, :__n_steps-1, :]
def testCustomizedAttention(self): batch_size = 2 max_time = 3 num_units = 2 memory = constant_op.constant([[[1., 1.], [2., 2.], [3., 3.]], [[4., 4.], [5., 5.], [6., 6.]]]) memory_sequence_length = constant_op.constant([3, 2]) attention_mechanism = wrapper.BahdanauAttention(num_units, memory, memory_sequence_length) # Sets all returned values to be all ones. def _customized_attention(unused_attention_mechanism, unused_cell_output, unused_attention_state, unused_attention_layer): """Customized attention. Returns: attention: `Tensor` of shape [batch_size, num_units], attention output. alignments: `Tensor` of shape [batch_size, max_time], sigma value for each input memory (prob. function of input keys). next_attention_state: A `Tensor` representing the next state for the attention. """ attention = array_ops.ones([batch_size, num_units]) alignments = array_ops.ones([batch_size, max_time]) next_attention_state = alignments return attention, alignments, next_attention_state attention_cell = wrapper.AttentionWrapper( rnn_cell.LSTMCell(2), attention_mechanism, attention_layer_size=None, # don't use attention layer. output_attention=False, alignment_history=(), attention_fn=_customized_attention, name='attention') self.assertEqual(num_units, attention_cell.output_size) initial_state = attention_cell.zero_state( batch_size=2, dtype=dtypes.float32) source_input_emb = array_ops.ones([2, 3, 2]) source_input_length = constant_op.constant([3, 2]) # 'state' is a tuple of # (cell_state, h, attention, alignments, alignment_history, attention_state) output, state = rnn.dynamic_rnn( attention_cell, inputs=source_input_emb, sequence_length=source_input_length, initial_state=initial_state, dtype=dtypes.float32) with self.session() as sess: sess.run(variables.global_variables_initializer()) output_value, state_value = sess.run([output, state], feed_dict={}) self.assertAllEqual(np.array([2, 3, 2]), output_value.shape) self.assertAllClose(np.array([[1., 1.], [1., 1.]]), state_value.attention) self.assertAllClose( np.array([[1., 1., 1.], [1., 1., 1.]]), state_value.alignments) self.assertAllClose( np.array([[1., 1., 1.], [1., 1., 1.]]), state_value.attention_state)
def crf_log_norm(inputs, sequence_lengths, transition_params): """Computes the normalization for a CRF. Args: inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials to use as input to the CRF layer. sequence_lengths: A [batch_size] vector of true sequence lengths. transition_params: A [num_tags, num_tags] transition matrix. Returns: log_norm: A [batch_size] vector of normalizers for a CRF. """ # Split up the first and rest of the inputs in preparation for the forward # algorithm. first_input = array_ops.slice(inputs, [0, 0, 0], [-1, 1, -1]) first_input = array_ops.squeeze(first_input, [1]) rest_of_input = array_ops.slice(inputs, [0, 1, 0], [-1, -1, -1]) # Compute the alpha values in the forward algorithm in order to get the # partition function. forward_cell = CrfForwardRnnCell(transition_params) _, alphas = rnn.dynamic_rnn( cell=forward_cell, inputs=rest_of_input, sequence_length=sequence_lengths - 1, initial_state=first_input, dtype=dtypes.float32) log_norm = math_ops.reduce_logsumexp(alphas, [1]) return log_norm
def testRNNWithKerasGRUCell(self): with self.cached_session() as sess: input_shape = 10 output_shape = 5 timestep = 4 batch = 100 (x_train, y_train), _ = testing_utils.get_test_data( train_samples=batch, test_samples=0, input_shape=(timestep, input_shape), num_classes=output_shape) y_train = keras.utils.to_categorical(y_train) cell = keras.layers.GRUCell(output_shape) inputs = array_ops.placeholder( dtypes.float32, shape=(None, timestep, input_shape)) predict = array_ops.placeholder( dtypes.float32, shape=(None, output_shape)) outputs, state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32) self.assertEqual(outputs.shape.as_list(), [None, timestep, output_shape]) self.assertEqual(state.shape.as_list(), [None, output_shape]) loss = losses.softmax_cross_entropy(predict, state) train_op = training.GradientDescentOptimizer(0.001).minimize(loss) sess.run([variables_lib.global_variables_initializer()]) _, outputs, state = sess.run( [train_op, outputs, state], {inputs: x_train, predict: y_train}) self.assertEqual(len(outputs), batch) self.assertEqual(len(state), batch)
def testTensorArrayStateIsAccepted(self): cell = TensorArrayStateRNNCell() in_graph_mode = context.in_graph_mode() if in_graph_mode: inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1)) else: inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32) with self.test_session() as sess: outputs, state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32, sequence_length=[4]) state = (state[0], state[1].stack()) if in_graph_mode: outputs, state = sess.run( [outputs, state], feed_dict={ inputs: [[[1], [2], [3], [4]]] }) if in_graph_mode: self.assertAllEqual(outputs, np.array([[[1], [2], [3], [4]]])) self.assertEqual(state[0], 4) self.assertAllEqual(state[1], np.array([[[1]], [[2]], [[3]], [[4]]])) else: self.assertAllEqual(outputs.numpy(), np.array([[[1], [2], [3], [4]]])) self.assertEqual(state[0].numpy(), 4) self.assertAllEqual(state[1].numpy(), np.array([[[1]], [[2]], [[3]], [[4]]]))
def testDeviceWrapperDynamicExecutionNodesAreAllProperlyLocated(self): if not test.is_gpu_available(): # Can't perform this test w/o a GPU return with self.test_session(use_gpu=True) as sess: with variable_scope.variable_scope( "root", initializer=init_ops.constant_initializer(0.5)): x = array_ops.zeros([1, 1, 3]) cell = rnn_cell_impl.DeviceWrapper(rnn_cell_impl.GRUCell(3), "/gpu:0") with ops.device("/cpu:0"): outputs, _ = rnn.dynamic_rnn( cell=cell, inputs=x, dtype=dtypes.float32) run_metadata = config_pb2.RunMetadata() opts = config_pb2.RunOptions( trace_level=config_pb2.RunOptions.FULL_TRACE) sess.run([variables_lib.global_variables_initializer()]) _ = sess.run(outputs, options=opts, run_metadata=run_metadata) step_stats = run_metadata.step_stats ix = 0 if "gpu" in step_stats.dev_stats[0].device else 1 gpu_stats = step_stats.dev_stats[ix].node_stats cpu_stats = step_stats.dev_stats[1 - ix].node_stats self.assertFalse([s for s in cpu_stats if "gru_cell" in s.node_name]) self.assertTrue([s for s in gpu_stats if "gru_cell" in s.node_name])
def testKerasAndTFRNNLayerOutputComparison(self): input_shape = 10 output_shape = 5 timestep = 4 batch = 20 (x_train, _), _ = testing_utils.get_test_data( train_samples=batch, test_samples=0, input_shape=(timestep, input_shape), num_classes=output_shape) fix_weights_generator = keras.layers.SimpleRNNCell(output_shape) fix_weights_generator.build((None, input_shape)) weights = fix_weights_generator.get_weights() with self.session(graph=ops_lib.Graph()) as sess: inputs = array_ops.placeholder( dtypes.float32, shape=(None, timestep, input_shape)) cell = keras.layers.SimpleRNNCell(output_shape) tf_out, tf_state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32) cell.set_weights(weights) [tf_out, tf_state] = sess.run([tf_out, tf_state], {inputs: x_train}) with self.session(graph=ops_lib.Graph()) as sess: k_input = keras.Input(shape=(timestep, input_shape), dtype=dtypes.float32) cell = keras.layers.SimpleRNNCell(output_shape) layer = keras.layers.RNN(cell, return_sequences=True, return_state=True) keras_out = layer(k_input) cell.set_weights(weights) k_out, k_state = sess.run(keras_out, {k_input: x_train}) self.assertAllClose(tf_out, k_out) self.assertAllClose(tf_state, k_state)
def _create_equivalent_canonical_rnn(self, cudnn_model, inputs, use_block_cell, scope="rnn"): if cudnn_model.rnn_mode is not "lstm": raise ValueError("%s is not supported!" % cudnn_model.rnn_mode) num_units = cudnn_model.num_units num_layers = cudnn_model.num_layers # To reuse cuDNN-trained models, must set # forget_bias, clip_cell = 0, False # In LSTMCell and LSTMBlockCell, forget_bias is added in addition to learned # bias, whereas cuDNN does not apply the additional bias. if use_block_cell: # pylint: disable=g-long-lambda single_cell = lambda: lstm_ops.LSTMBlockCell(num_units, forget_bias=0, clip_cell=False) # pylint: enable=g-long-lambda else: single_cell = lambda: rnn_cell_impl.LSTMCell(num_units, forget_bias=0) cell = rnn_cell_impl.MultiRNNCell( [single_cell() for _ in range(num_layers)]) return rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32, time_major=True, scope=scope)
def _create_multi_lstm_cell_ops(batch_size, num_units, input_depth, num_layers, max_time, compiled): with variable_scope.variable_scope( "root", initializer=init_ops.random_uniform_initializer(-0.1, 0.1, seed=2)): inputs = variable_scope.get_variable( "inputs", initializer=random_ops.random_uniform( (max_time, batch_size, input_depth), seed=1)) maybe_xla = lambda c: rnn_cell.CompiledWrapper(c) if compiled else c cell = core_rnn_cell_impl.MultiRNNCell( [maybe_xla(core_rnn_cell_impl.LSTMCell(num_units)) for _ in range(num_layers)]) initial_state = cell.zero_state( batch_size=batch_size, dtype=dtypes.float32) outputs, final_state = rnn.dynamic_rnn( cell=cell, inputs=inputs, initial_state=initial_state, time_major=True) flat_final_state = nest.flatten(final_state) trainable_variables = variables.trainable_variables() outputs_grad = gradients_impl.gradients( [outputs], trainable_variables + [inputs] + nest.flatten(initial_state)) final_state_grad = gradients_impl.gradients( flat_final_state, trainable_variables + [inputs] + nest.flatten(initial_state)) return {"outputs": outputs, "final_state": flat_final_state, "outputs_grad": outputs_grad, "final_state_grad": final_state_grad}
def RNN(inputs, lens, name, reuse): print ("Building network " + name) # Define weights weights = tf.Variable(tf.random_normal([__n_hidden, n_output]), name=name+"_weights") biases = tf.Variable(tf.random_normal([n_output]), name=name+"_biases") # Define a lstm cell with tensorflow outputs, states = rnn.dynamic_rnn( __cell_kind(__n_hidden), inputs, sequence_length=lens, dtype=tf.float32, scope=name, time_major=False) assert outputs.get_shape() == (__batch_size, __n_steps, __n_hidden) print ("Done building network " + name) # # All these asserts are actually documentation: they can't be out of date # outputs = tf.expand_dims(outputs, 2) assert outputs.get_shape() == (__batch_size, __n_steps, 1, __n_hidden) tiled_weights = tf.tile(tf.expand_dims(tf.expand_dims(weights, 0), 0), [__batch_size, __n_steps, 1, 1]) assert tiled_weights.get_shape() == (__batch_size, __n_steps, __n_hidden, n_output) #assert tiled_weights.get_shape() == (1, 1, __n_hidden, n_output) # Linear activation, using rnn inner loop output for each char finals = tf.batch_matmul(outputs, tiled_weights) + biases assert finals.get_shape() == (__batch_size, __n_steps, 1, n_output) return tf.squeeze(finals)
def __call__(self, inputs, initial_state=None, dtype=None, sequence_length=None, scope=None): is_list = isinstance(inputs, list) if self._use_dynamic_rnn: if is_list: inputs = array_ops.pack(inputs) outputs, state = rnn.dynamic_rnn( self._cell, inputs, sequence_length=sequence_length, initial_state=initial_state, dtype=dtype, time_major=True, scope=scope) if is_list: # Convert outputs back to list outputs = array_ops.unpack(outputs) else: # non-dynamic rnn if not is_list: inputs = array_ops.unpack(inputs) outputs, state = rnn.rnn(self._cell, inputs, initial_state=initial_state, dtype=dtype, sequence_length=sequence_length, scope=scope) if not is_list: # Convert outputs back to tensor outputs = array_ops.pack(outputs) return outputs, state
def bidirectional_rnn(self, cell, inputs, lengths, scope=None): name = scope.name or "BiRNN" # Forward direction with vs.variable_scope(name + "_FW") as fw_scope: output_fw, output_state_fw = rnn.dynamic_rnn(cell, inputs, time_major=True, dtype=dtypes.float32, sequence_length=lengths, scope=fw_scope) # Backward direction with vs.variable_scope(name + "_BW") as bw_scope: output_bw, output_state_bw = rnn.dynamic_rnn(cell, inputs, time_major=True, dtype=dtypes.float32, sequence_length=lengths, scope=bw_scope) output_bw = tf.reverse_sequence(output_bw, tf.to_int64(lengths), seq_dim=0, batch_dim=1) outputs = output_fw + output_bw output_state = output_state_fw + output_state_bw return (outputs, output_state)
def ndlstm_base_dynamic(inputs, noutput, scope=None, reverse=False): """Run an LSTM, either forward or backward. This is a 1D LSTM implementation using dynamic_rnn and the TensorFlow LSTM op. Args: inputs: input sequence (length, batch_size, ninput) noutput: depth of output scope: optional scope name reverse: run LSTM in reverse Returns: Output sequence (length, batch_size, noutput) """ with variable_scope.variable_scope(scope, "SeqLstm", [inputs]): # TODO(tmb) make batch size, sequence_length dynamic # example: sequence_length = tf.shape(inputs)[0] _, batch_size, _ = _shape(inputs) lstm_cell = core_rnn_cell_impl.BasicLSTMCell(noutput, state_is_tuple=False) state = array_ops.zeros([batch_size, lstm_cell.state_size]) sequence_length = int(inputs.get_shape()[0]) sequence_lengths = math_ops.to_int64( array_ops.fill([batch_size], sequence_length)) if reverse: inputs = array_ops.reverse_v2(inputs, [0]) outputs, _ = rnn.dynamic_rnn( lstm_cell, inputs, sequence_lengths, state, time_major=True) if reverse: outputs = array_ops.reverse_v2(outputs, [0]) return outputs
def _testDropoutWrapper(self, batch_size=None, time_steps=None, parallel_iterations=None, **kwargs): with self.test_session() as sess: with variable_scope.variable_scope( "root", initializer=init_ops.constant_initializer(0.5)): if batch_size is None and time_steps is None: # 2 time steps, batch size 1, depth 3 batch_size = 1 time_steps = 2 x = constant_op.constant( [[[2., 2., 2.]], [[1., 1., 1.]]], dtype=dtypes.float32) m = rnn_cell_impl.LSTMStateTuple( *[constant_op.constant([[0.1, 0.1, 0.1]], dtype=dtypes.float32) ] * 2) else: x = constant_op.constant( np.random.randn(time_steps, batch_size, 3).astype(np.float32)) m = rnn_cell_impl.LSTMStateTuple(*[ constant_op.constant( [[0.1, 0.1, 0.1]] * batch_size, dtype=dtypes.float32) ] * 2) outputs, final_state = rnn.dynamic_rnn( cell=rnn_cell_impl.DropoutWrapper( rnn_cell_impl.LSTMCell(3), dtype=x.dtype, **kwargs), time_major=True, parallel_iterations=parallel_iterations, inputs=x, initial_state=m) sess.run([variables_lib.global_variables_initializer()]) res = sess.run([outputs, final_state]) self.assertEqual(res[0].shape, (time_steps, batch_size, 3)) self.assertEqual(res[1].c.shape, (batch_size, 3)) self.assertEqual(res[1].h.shape, (batch_size, 3)) return res
def benchmarkLSTMBlockCellBpropWithDynamicRNN(self): print("BlockLSTMCell backward propagation via dynamic_rnn().") print("--------------------------------------------------------------") print("LSTMBlockCell Seconds per inference.") print("batch_size,cell_size,input_size,time_steps,use_gpu,wall_time") iters = 10 for config in benchmarking.dict_product({ "batch_size": [1, 8, 13, 32, 67, 128], "cell_size": [128, 250, 512, 650, 1024, 1350], "time_steps": [40], "use_gpu": [True, False] }): with ops.Graph().as_default(): with benchmarking.device(use_gpu=config["use_gpu"]): time_steps = config["time_steps"] batch_size = config["batch_size"] cell_size = input_size = config["cell_size"] inputs = variable_scope.get_variable( "x", [time_steps, batch_size, cell_size], trainable=False, dtype=dtypes.float32) with variable_scope.variable_scope( "rnn", reuse=variable_scope.AUTO_REUSE): w = variable_scope.get_variable( "rnn/lstm_cell/kernel", shape=[input_size + cell_size, cell_size * 4], dtype=dtypes.float32) b = variable_scope.get_variable( "rnn/lstm_cell/bias", shape=[cell_size * 4], dtype=dtypes.float32, initializer=init_ops.zeros_initializer()) cell = lstm_ops.LSTMBlockCell(cell_size) outputs = rnn.dynamic_rnn( cell, inputs, time_major=True, dtype=dtypes.float32) grads = gradients_impl.gradients(outputs, [inputs, w, b]) init_op = variables.global_variables_initializer() with session.Session() as sess: sess.run(init_op) wall_time = benchmarking.seconds_per_run(grads, sess, iters) # Print to stdout. If the TEST_REPORT_FILE_PREFIX environment variable # is set, this will produce a copy-paste-able CSV file. print(",".join( map(str, [ batch_size, cell_size, cell_size, time_steps, config["use_gpu"], wall_time ]))) benchmark_name_template = "_".join([ "LSTMBlockCell_bprop", "BS%(batch_size)i", "CS%(cell_size)i", "IS%(cell_size)i", "TS%(time_steps)i", "gpu_%(use_gpu)s" ]) self.report_benchmark( name=benchmark_name_template % config, iters=iters, wall_time=wall_time, extras=config)
def testBatchSizeFromInput(self): cell = Plus1RNNCell() in_graph_mode = context.in_graph_mode() # With static batch size if in_graph_mode: inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5)) initial_state = array_ops.placeholder(dtypes.float32, shape=(3, 5)) else: inputs = np.zeros((3, 4, 5), dtype=np.float32) initial_state = np.zeros((3, 5), dtype=np.float32) # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) if in_graph_mode: self.assertEqual(3, outputs.shape[0].value) self.assertEqual(3, state.shape[0].value) else: self.assertEqual(3, outputs.shape[0]) self.assertEqual(3, state.shape[0]) # - With initial_state outputs, state = rnn.dynamic_rnn( cell, inputs, initial_state=initial_state) if in_graph_mode: self.assertEqual(3, outputs.shape[0].value) self.assertEqual(3, state.shape[0].value) else: self.assertEqual(3, outputs.shape[0]) self.assertEqual(3, state.shape[0]) # Without static batch size # Tensor shapes are fully determined in Eager mode, so only run this # test in graph mode. if in_graph_mode: inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5)) # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) self.assertEqual(None, outputs.shape[0].value) self.assertEqual(None, state.shape[0].value) # - With initial_state outputs, state = rnn.dynamic_rnn( cell, inputs, initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5))) self.assertEqual(None, outputs.shape[0].value) self.assertEqual(None, state.shape[0].value)
def _multi_seq_fn(): """Decoding of highest scoring sequence.""" # For simplicity, in shape comments, denote: # 'batch_size' by 'B', 'max_seq_len' by 'T' , 'num_tags' by 'O' (output). num_tags = potentials.get_shape()[2].value # Computes forward decoding. Get last score and backpointers. crf_fwd_cell = CrfDecodeForwardRnnCell(transition_params) initial_state = array_ops.slice(potentials, [0, 0, 0], [-1, 1, -1]) initial_state = array_ops.squeeze(initial_state, axis=[1]) # [B, O] inputs = array_ops.slice(potentials, [0, 1, 0], [-1, -1, -1]) # [B, T-1, O] # Sequence length is not allowed to be less than zero. sequence_length_less_one = math_ops.maximum( constant_op.constant(0, dtype=sequence_length.dtype), sequence_length - 1) backpointers, last_score = rnn.dynamic_rnn( # [B, T - 1, O], [B, O] crf_fwd_cell, inputs=inputs, sequence_length=sequence_length_less_one, initial_state=initial_state, time_major=False, dtype=dtypes.int32) backpointers = gen_array_ops.reverse_sequence( # [B, T - 1, O] backpointers, sequence_length_less_one, seq_dim=1) # Computes backward decoding. Extract tag indices from backpointers. crf_bwd_cell = CrfDecodeBackwardRnnCell(num_tags) initial_state = math_ops.cast(math_ops.argmax(last_score, axis=1), # [B] dtype=dtypes.int32) initial_state = array_ops.expand_dims(initial_state, axis=-1) # [B, 1] decode_tags, _ = rnn.dynamic_rnn( # [B, T - 1, 1] crf_bwd_cell, inputs=backpointers, sequence_length=sequence_length_less_one, initial_state=initial_state, time_major=False, dtype=dtypes.int32) decode_tags = array_ops.squeeze(decode_tags, axis=[2]) # [B, T - 1] decode_tags = array_ops.concat([initial_state, decode_tags], # [B, T] axis=1) decode_tags = gen_array_ops.reverse_sequence( # [B, T] decode_tags, sequence_length, seq_dim=1) best_score = math_ops.reduce_max(last_score, axis=1) # [B] return decode_tags, best_score
def testInvalidDtype(self): if context.executing_eagerly(): inputs = np.zeros((3, 4, 5), dtype=np.int32) else: inputs = array_ops.placeholder(dtypes.int32, shape=(3, 4, 5)) cells = [ rnn_cell_impl.BasicRNNCell, rnn_cell_impl.GRUCell, rnn_cell_impl.BasicLSTMCell, rnn_cell_impl.LSTMCell, ] for cell_cls in cells: with self.cached_session(): with self.assertRaisesRegexp( ValueError, "RNN cell only supports floating"): cell = cell_cls(2, dtype=dtypes.int32) rnn.dynamic_rnn(cell, inputs, dtype=dtypes.int32)
def testScalarStateIsAccepted(self): cell = ScalarStateRNNCell() inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1)) with self.test_session() as sess: outputs, state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32, sequence_length=[4]) outputs, state = sess.run( [outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]}) self.assertAllEqual(outputs, [[[1], [2], [3], [4]]]) self.assertEqual(state, 4)
def __init__(self, max_words, num_classes, vocab_size, embedding_size, num_hidden): # input, output, dropout placeholders self.text = tf.placeholder(tf.int32, [None, max_words], name="input_text") self.extra = tf.placeholder(tf.int32, [None, max_words], name="input_extra") self.output = tf.placeholder(tf.float32, [None, num_classes], name="output_y") self.sequence_lengths = tf.placeholder(tf.int32, [None], name="sequence_lengths") self.dropout_prob = tf.placeholder(tf.float32, name="dropout_probability") # Word embedding layer with tf.device("/cpu:0"), tf.name_scope("word_embedding"): embedding_matrix = tf.Variable( tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0), # random numbers between -1 and 1 name="embedding_matrix") self.lookup = tf.nn.embedding_lookup(embedding_matrix, self.text) # GRU with tf.name_scope("GRU"): output, state = rnn.dynamic_rnn( rnn_cell.GRUCell(num_hidden), self.lookup, dtype=tf.float32, sequence_length=self.sequence_lengths) output = tf.transpose(output, [1, 0, 2]) self.gru = tf.gather(output, int(output.get_shape()[0]) - 1) # Add dropout with tf.name_scope("dropout"): self.dropout = tf.nn.dropout(self.gru, self.dropout_prob) # add in extra data and relu layer with tf.name_scope("extra_data"): combined = tf.concat(1, [self.dropout, self.extra]) weights_e = tf.Variable(tf.truncated_normal([num_hidden, num_hidden], stddev=0.1), name="weights_extra") biases_e = tf.Variable(tf.constant(0.1, shape=[num_hidden]), name="biases_extra") processed = tf.relu(tf.matmul(combined, weights_e) + biases_e) # Final output with tf.name_scope("output"): weights = tf.Variable(tf.truncated_normal([num_hidden, num_classes], stddev=0.1), name="weights") biases = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="biases") unscaled = tf.matmul(processed, weights) + biases self.scores = tf.nn.softmax(unscaled, name="scores") self.predictions = tf.argmax(self.scores, dimension=1, name="predictions") # calculate loss with tf.name_scope("loss"): losses = tf.nn.softmax_cross_entropy_with_logits(unscaled, self.output) self.loss = tf.reduce_mean(losses) # calculate accuracy with tf.name_scope("accuracy"): correct_predictions = tf.equal(self.predictions, tf.argmax(self.output, 1)) self.accuracy = 100 * tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
def testSimpleRNNCellAndBasicRNNCellComparison(self): input_shape = 10 output_shape = 5 timestep = 4 batch = 20 (x_train, _), _ = testing_utils.get_test_data( train_samples=batch, test_samples=0, input_shape=(timestep, input_shape), num_classes=output_shape) fix_weights_generator = keras.layers.SimpleRNNCell(output_shape) fix_weights_generator.build((None, input_shape)) # The SimpleRNNCell contains 3 weights: kernel, recurrent_kernel, and bias # The BasicRNNCell contains 2 weight: kernel and bias, where kernel is # zipped [kernel, recurrent_kernel] in SimpleRNNCell. keras_weights = fix_weights_generator.get_weights() kernel, recurrent_kernel, bias = keras_weights tf_weights = [np.concatenate((kernel, recurrent_kernel)), bias] with self.session(graph=ops_lib.Graph()) as sess: inputs = array_ops.placeholder( dtypes.float32, shape=(None, timestep, input_shape)) cell = keras.layers.SimpleRNNCell(output_shape) k_out, k_state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32) cell.set_weights(keras_weights) [k_out, k_state] = sess.run([k_out, k_state], {inputs: x_train}) with self.session(graph=ops_lib.Graph()) as sess: inputs = array_ops.placeholder( dtypes.float32, shape=(None, timestep, input_shape)) cell = rnn_cell_impl.BasicRNNCell(output_shape) tf_out, tf_state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32) cell.set_weights(tf_weights) [tf_out, tf_state] = sess.run([tf_out, tf_state], {inputs: x_train}) self.assertAllClose(tf_out, k_out, atol=1e-5) self.assertAllClose(tf_state, k_state, atol=1e-5)
def testBatchSizeFromInput(self): cell = Plus1RNNCell() in_eager_mode = context.executing_eagerly() # With static batch size if in_eager_mode: inputs = np.zeros((3, 4, 5), dtype=np.float32) initial_state = np.zeros((3, 5), dtype=np.float32) else: inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5)) initial_state = array_ops.placeholder(dtypes.float32, shape=(3, 5)) # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) self.assertEqual(3, outputs.shape[0]) self.assertEqual(3, state.shape[0]) # - With initial_state outputs, state = rnn.dynamic_rnn( cell, inputs, initial_state=initial_state) self.assertEqual(3, outputs.shape[0]) self.assertEqual(3, state.shape[0]) # Without static batch size # Tensor shapes are fully determined with eager execution enabled, # so only run this test for graph construction. if not in_eager_mode: inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5)) # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) self.assertEqual(None, outputs.shape.dims[0].value) self.assertEqual(None, state.shape.dims[0].value) # - With initial_state outputs, state = rnn.dynamic_rnn( cell, inputs, initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5))) self.assertEqual(None, outputs.shape.dims[0].value) self.assertEqual(None, state.shape.dims[0].value)
def time_aware_gru_net(self, hidden_units, input_data, input_length, type): if type == 'T-SeqRec': cell = self.build_time_aware_gru_cell_sigmoid(hidden_units) elif type == 'new': cell = self.build_time_aware_gru_cell_new(hidden_units) elif type == 'T_Gru_Extend': cell = self.build_time_aware_gru_cell_extend(hidden_units) #cell = self.build_cell(hidden_units) self.input_length = tf.reshape(input_length, [-1]) outputs, _ = dynamic_rnn(cell, inputs=input_data, sequence_length=self.input_length, dtype=tf.float32) return outputs
def force(): force_embed = tf.nn.embedding_lookup(embedding_tensor, force_tensor[:, :-1]) decoder_input = tf.concat([ input_tensor, tf.concat([tf.tile(init_tensor, [batch_size, 1, 1]), force_embed], 1) ], 2) decoder_input = tf.nn.dropout(decoder_input, keep_prob=keep_prob) force_decoder, _ = dynamic_rnn(decoder_cell, decoder_input, sequence_length=sequence_length, dtype=tf.float32) return force_decoder
def add_lstm(self, inputs): if 'lstm_conf' in self.nn_config: lstm_conf = self.nn_config['lstm_conf'] activation = None if lstm_conf['batch_norm'] else lstm_conf[ 'lstm_activation'] return_seq = True if '1dCNN' in self.nn_config else False cell = LSTMCell(units=lstm_conf['lstm_units'], activation=activation) if lstm_conf['method'] == 'dynamic_rnn': rnn_outputs1, states = dynamic_rnn(cell, inputs, dtype=tf.float32) lstm_outputs = tf.reshape(rnn_outputs1[:, -1, :], [-1, lstm_conf['lstm_units']]) elif lstm_conf['method'] == 'keras_lstm_layer': lstm_outputs = LSTM(lstm_conf['lstm_units'], name="first_lstm", activation=activation, input_shape=(self.data_config['lookback'], self.ins), return_sequences=return_seq)(inputs) else: rnn_layer = RNN(cell, return_sequences=return_seq) lstm_outputs = rnn_layer(inputs) # [batch_size, neurons] if self.verbose > 0: print(lstm_outputs.shape, 'before reshaping', K.eval(tf.rank(lstm_outputs))) lstm_outputs = tf.reshape(lstm_outputs[:, :], [-1, lstm_conf['lstm_units']]) if self.verbose > 0: print(lstm_outputs.shape, 'after reshaping', K.eval(tf.rank(lstm_outputs))) if lstm_conf['batch_norm']: rnn_outputs3 = BatchNormalization()(lstm_outputs) lstm_outputs = Activation('relu')(rnn_outputs3) if lstm_conf['dropout'] is not None: lstm_outputs = Dropout(lstm_conf['dropout'])(lstm_outputs) else: lstm_outputs = inputs return lstm_outputs
def setup_decoder(self): self.decoder_state_output =[] if self.num_layers > 1: self.decoder_cell = rnn_cell.GRUCell(self.size) # self.decoder_cell = tf.contrib.rnn.GRUCell(self.size,kernel_initializer = tf.contrib.layers.xavier_initializer(dtype=tf.float32)) self.attn_cell = GRUCellAttn(self.size, self.encoder_output, scope="DecoderAttnCell") with vs.variable_scope("Decoder"): inp = self.decoder_inputs for i in xrange(self.num_layers - 1): with vs.variable_scope("DecoderCell%d" % i) as scope: out, state_output = rnn.dynamic_rnn(self.decoder_cell, inp, time_major=True, dtype=dtypes.float32, sequence_length=self.target_length, scope=scope) inp = self.dropout(out) self.decoder_state_output.append(state_output) with vs.variable_scope("DecoderAttnCell") as scope: out, state_output = rnn.dynamic_rnn(self.attn_cell, inp, time_major=True, dtype=dtypes.float32, sequence_length=self.target_length, scope=scope) self.decoder_output = self.dropout(out) self.decoder_state_output.append(state_output)
def define_two_layer_LSTM(inputX, seqLengths, nHidden): lstm_cell = rnn_cell.LSTMCell(nHidden, state_is_tuple=True) cell = rnn_cell.MultiRNNCell([lstm_cell] * 2) initial = cell.zero_state(tf.shape(inputX)[0], tf.float32) outputs, _ = dynamic_rnn(cell, inputX, dtype=tf.float32, sequence_length=seqLengths, initial_state=initial, time_major=False) return outputs, nHidden # hidden*2 is number of actual hidden states.
def model(self): """ :param x: inputs of size [T, batch_size, input_size] :param W: matrix of fully-connected output layer weights :param b: vector of fully-connected output layer biases """ cell = rnn_cell.BasicLSTMCell(self.hidden_dim) outputs, states = rnn.dynamic_rnn(cell, self.x, dtype=tf.float32) num_examples = tf.shape(self.x)[0] W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1]) out = tf.batch_matmul(outputs, W_repeated) + self.b_out out = tf.squeeze(out) return out
def benchmarkLSTMBlockCellFpropWithDynamicRNN(self): print("BlockLSTMCell forward propagation via dynamic_rnn().") print("--------------------------------------------------------------") print("LSTMBlockCell Seconds per inference.") print("batch_size,cell_size,input_size,time_steps,use_gpu,wall_time") iters = 10 for config in benchmarking.dict_product({ "batch_size": [1, 32, 128], "cell_size": [32, 128, 512], "input_size": [128, 512], "time_steps": [10, 25, 100], "use_gpu": [True, False] }): with ops.Graph().as_default(): with benchmarking.device(use_gpu=config["use_gpu"]): inputs = variable_scope.get_variable( "x", [ config["time_steps"], config["batch_size"], config["input_size"] ]) cell = lstm_ops.LSTMBlockCell(config["cell_size"]) outputs = rnn.dynamic_rnn(cell, inputs, time_major=True, dtype=dtypes.float32) init_op = variables.global_variables_initializer() with session.Session() as sess: sess.run(init_op) wall_time = benchmarking.seconds_per_run( outputs, sess, iters) # Print to stdout. If the TEST_REPORT_FILE_PREFIX environment variable # is set, this will produce a copy-paste-able CSV file. print(",".join( map(str, [ config["batch_size"], config["cell_size"], config["input_size"], config["time_steps"], config["use_gpu"], wall_time ]))) benchmark_name_template = "_".join([ "LSTMBlockCell_fprop", "BS%(batch_size)i", "CS%(cell_size)i", "IS%(input_size)i", "TS%(time_steps)i", "gpu_%(use_gpu)s" ]) self.report_benchmark(name=benchmark_name_template % config, iters=iters, wall_time=wall_time, extras=config)
def stacked_lstm(cell_fn, input_tensor, num_cells, num_lstm_layers=1, return_only_last_output=True): (x, t) = input_tensor for i in range(num_lstm_layers): x, _ = dynamic_rnn(cell=cell_fn(num_cells), inputs=(x, t), dtype=tf.float32, scope='LSTM_' + str(i)) if return_only_last_output: return tf.squeeze(x[:, -1, :]) return x
def modified_grnn_net(self, hidden_units, input_data, input_length, scope='modified_grnn'): cell = self.build_modified_grnn_cell(hidden_units) self.input_length = tf.reshape(input_length, [-1]) outputs, _ = dynamic_rnn(cell, inputs=input_data, sequence_length=self.input_length, dtype=tf.float32, scope=scope) return outputs
def _multi_seq_fn(): """Forward computation of alpha values.""" rest_of_input = array_ops.slice(inputs, [0, 1, 0], [-1, -1, -1]) # Compute the alpha values in the forward algorithm in order to get the # partition function. forward_cell = CrfForwardRnnCell(transition_params) _, alphas = rnn.dynamic_rnn(cell=forward_cell, inputs=rest_of_input, sequence_length=sequence_lengths - 1, initial_state=first_input, dtype=dtypes.float32) log_norm = math_ops.reduce_logsumexp(alphas, [1]) return log_norm
def __init__(self, num_symbols, num_embed_units, num_units, num_labels, embed, learning_rate=0.001, max_gradient_norm=5.0): self.texts = tf.placeholder(tf.int32, [None, None]) # shape: sentence*max_word self.text_length = tf.placeholder(tf.int32, [None]) # shape: sentence self.labels = tf.placeholder(tf.int32, [None]) # shape: sentence self.keep_prob = tf.placeholder(tf.float32) self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32) self.global_step = tf.Variable(0, trainable=False) self.epoch = tf.Variable(0, trainable=False) self.epoch_add_op = self.epoch.assign(self.epoch + 1) # build the embedding table (index to vector) self.embed = tf.get_variable('embed', dtype=tf.float32, initializer=embed) self.embed_inputs = tf.nn.embedding_lookup(self.embed, self.texts) # shape: sentence*max_word*num_embed_units fw_cell = DropoutWrapper(BasicLSTMCell(num_units), output_keep_prob=self.keep_prob) bw_cell = DropoutWrapper(BasicLSTMCell(num_units), output_keep_prob=self.keep_prob) middle_outputs, middle_states = bidirectional_dynamic_rnn(fw_cell, bw_cell, self.embed_inputs, self.text_length, dtype=tf.float32, scope="word_rnn") middle_outputs = tf.concat(middle_outputs, 2) # shape: sentence*max_word*(2*num_units) middle_inputs = tf.expand_dims(tf.reduce_max(middle_outputs, axis=1), 0) # shape: 1*sentence*(2*num_units) top_cell = DropoutWrapper(BasicLSTMCell(num_units), output_keep_prob=self.keep_prob) outputs, states = dynamic_rnn(top_cell, middle_inputs, dtype=tf.float32, scope="sentence_rnn") self.outputs = outputs[0] # shape: sentence*num_units logits = tf.layers.dense(self.outputs, num_labels) self.loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.labels, logits=logits), name='loss') mean_loss = self.loss / tf.cast(tf.shape(self.labels)[0], dtype=tf.float32) self.predict_labels = tf.argmax(logits, 1, 'predict_labels', output_type=tf.int32) self.accuracy = tf.reduce_sum(tf.cast(tf.equal(self.labels, self.predict_labels), tf.int32), name='accuracy') self.params = tf.trainable_variables() # calculate the gradient of parameters opt = tf.train.AdamOptimizer(self.learning_rate) gradients = tf.gradients(mean_loss, self.params) clipped_gradients, self.gradient_norm = tf.clip_by_global_norm(gradients, max_gradient_norm) self.update = opt.apply_gradients(zip(clipped_gradients, self.params), global_step=self.global_step) self.saver = tf.train.Saver(max_to_keep=3, pad_step_number=True)
def setup_decoder(self): # decoder的设置 # vs.variable_scope:变量命名,使得参数保持一致,https://morvanzhou.github.io/tutorials/machine-learning/tensorflow/5-12-scope/ with vs.variable_scope("Decoder"): inp = tf.nn.dropout(self.decoder_inputs, self.keep_prob) if self.num_layers > 1: with vs.variable_scope("RNN"): # 理解成调用RNN decoder_cell = rnn_cell.GRUCell(self.size) decoder_cell = rnn_cell.DropoutWrapper( decoder_cell, output_keep_prob=self.keep_prob) self.decoder_cell = rnn_cell.MultiRNNCell( [decoder_cell] * (self.num_layers - 1), state_is_tuple=True) inp, _ = rnn.dynamic_rnn( self.decoder_cell, inp, self.tgt_len, dtype=tf.float32, time_major=True, initial_state=self.decoder_cell.zero_state( self.batch_size, dtype=tf.float32)) with vs.variable_scope("Attn"): # 理解为调用attention self.attn_cell = GRUCellAttn(self.size, self.len_inp, self.encoder_output, self.src_mask, self.decode_method) # 设置 attention self.decoder_output, _ = rnn.dynamic_rnn( self.attn_cell, inp, self.tgt_len, dtype=tf.float32, time_major=True, initial_state=self.attn_cell.zero_state( self.batch_size, dtype=tf.float32, ))
def BuildFullModel(): """Build the full model with conv,rnn,opt.""" seq = [] for i in range(4): with variable_scope.variable_scope('inp_%d' % i): seq.append(array_ops.reshape(BuildSmallModel(), [2, 1, -1])) cell = rnn_cell.BasicRNNCell(16) out = rnn.dynamic_rnn( cell, array_ops.concat(seq, axis=1), dtype=dtypes.float32)[0] target = array_ops.ones_like(out) loss = nn_ops.l2_loss(math_ops.reduce_mean(target - out)) sgd_op = gradient_descent.GradientDescentOptimizer(1e-2) return sgd_op.minimize(loss)
def _make_graph(self): x = tf.placeholder("float", [self.batch_size, self.seq_len, self.num_nodes, self.input_dim], name='input') y = tf.placeholder("float", [self.batch_size, self.num_nodes, self.output_dim], name='label') self.input = x self.label = y x = tf.reshape(x, [self.batch_size, self.seq_len, self.num_nodes * self.input_dim]) outputs, state = dynamic_rnn(self.cell, x, dtype=tf.float32) outputs = tf.transpose(outputs, [1, 0, 2]) pred = tf.contrib.layers.fully_connected(outputs[-1], self.num_nodes * self.output_dim, activation_fn=None) # Define loss and optimizer pred = tf.reshape(pred, [self.batch_size, self.num_nodes, self.output_dim], name='predictions') self.prediction = pred
def generate_signal(self, signal_key, context, **kwargs): if signal_key == "loss": obs = context.get_signal('obs') get_actions_cell = GetActionsCell(self.policy) initial_state = get_actions_cell.zero_state(tf.shape(obs)[1], tf.float32) actions, _ = dynamic_rnn( get_actions_cell, obs, initial_state=initial_state, parallel_iterations=1, swap_memory=False, time_major=True) loss = self.env.build_trajectory_loss(actions, obs) return loss else: raise Exception("NotImplemented")
def prediction(self): # Recurrent network. network = rnn_cell.GRUCell(self._num_hidden) network = rnn_cell.DropoutWrapper(network, output_keep_prob=self.dropout) network = rnn_cell.MultiRNNCell([network] * self._num_layers) output, _ = rnn.dynamic_rnn(network, data, dtype=tf.float32) # Select last output. output = tf.transpose(output, [1, 0, 2]) last = tf.gather(output, int(output.get_shape()[0]) - 1) # Softmax layer. weight, bias = self._weight_and_bias(self._num_hidden, int(self.target.get_shape()[1])) prediction = tf.nn.softmax(tf.matmul(last, weight) + bias) return prediction
def __call__(self, inputs, seq_len=None): if self.call_cnt == 0: self.cell = LSTMCell( self.output_dim, initializer=self.initializer(dtype=inputs.dtype)) with tf.variable_scope(self.scope) as scope: self.check_reuse(scope) #if self.call_cnt ==0: #self.cell = LSTMCell(self.output_dim,initializer = self.initializer) #cell = BasicLSTMCell(self.output_dim) return rnn.dynamic_rnn(self.cell, inputs, seq_len, dtype=inputs.dtype)
def rnn_model(data): feature_columns_count = 3 layer = { 'weights': tf.Variable(tf.random_uniform([rnn_size, 1], -1, 1)), 'biases': tf.Variable(tf.random_uniform([1])) } length = [feature_columns_count for _ in range(batch_size)] lstm_cell = rnn_cell.LSTMCell(rnn_size) outputs, _ = rnn.dynamic_rnn(lstm_cell, data, dtype=tf.float32, sequence_length=length) output = tf.matmul(last_relevant(outputs, feature_columns_count), layer['weights']) + layer['biases'] return output
def __init__(self, nact, rnn_units=256): cells = [GRUCell(rnn_units, kernel_initializer=orthogonal()) for _ in range(2)] self.gru = MultiRNNCell(cells) self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32) self.obs_ph = tf.placeholder(dtype=tf.float32) fc1 = dense(self.obs_ph, rnn_units, activation=elu, kernel_initializer=orthogonal(), name='fc1') expand = tf.expand_dims(fc1, axis=0, name='expand') rnn_out, self.state = dynamic_rnn(self.gru, expand, initial_state=self.state) reshape = tf.reshape(rnn_out, shape=[-1, rnn_units], name='reshape') self.logits = dense(reshape, nact, kernel_initializer=orthogonal(0.01), name='logits') self.pi = tf.nn.softmax(self.logits, name='pi') self.action = boltzmann(self.pi) self.value = dense(self.logits, 1, kernel_initializer=orthogonal(), name='value')
def _construct_rnn(self, features): """Apply an RNN to `features`. The `features` dict must contain `self._inputs_key`, and the corresponding input should be a `Tensor` of shape `[batch_size, padded_length, k]` where `k` is the dimension of the input for each element of a sequence. `activations` has shape `[batch_size, sequence_length, n]` where `n` is `self._target_column.num_label_columns`. In the case of a multiclass classifier, `n` is the number of classes. `final_state` has shape determined by `self._cell` and its dtype must match `self._dtype`. Args: features: a `dict` containing the input for the RNN and (optionally) an initial state and information about sequence lengths. Returns: activations: the output of the RNN, projected to the appropriate number of dimensions. final_state: the final state output by the RNN. Raises: KeyError: if `features` does not contain `self._inputs_key`. """ with ops.name_scope('RNN'): inputs = features.get(self._inputs_key) if inputs is None: raise KeyError('features must contain the key {}'.format( self._inputs_key)) if inputs.dtype != self._dtype: inputs = math_ops.cast(inputs, self._dtype) initial_state = features.get(self._initial_state_key) rnn_outputs, final_state = rnn.dynamic_rnn( cell=self._cell, inputs=inputs, initial_state=initial_state, dtype=self._dtype, parallel_iterations=self._parallel_iterations, swap_memory=self._swap_memory, time_major=False) activations = layers.fully_connected( inputs=rnn_outputs, num_outputs=self._target_column.num_label_columns, activation_fn=None, trainable=False) return activations, final_state
def testDynamicDecodeRNNWithBasicTrainingSamplerMatchesDynamicRNN(self): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 cell_depth = 10 max_out = max(sequence_length) with self.test_session() as sess: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) zero_state = cell.zero_state(dtype=dtypes.float32, batch_size=batch_size) sampler = sampling_decoder.BasicTrainingSampler( inputs, sequence_length) my_decoder = sampling_decoder.BasicSamplingDecoder( cell=cell, sampler=sampler, initial_state=zero_state) # Match the variable scope of dynamic_rnn below so we end up # using the same variables with vs.variable_scope("rnn"): final_decoder_outputs, final_decoder_state = decoder.dynamic_decode_rnn( my_decoder) with vs.variable_scope(vs.get_variable_scope(), reuse=True): final_rnn_outputs, final_rnn_state = rnn.dynamic_rnn( cell, inputs, sequence_length=sequence_length, initial_state=zero_state) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "final_decoder_outputs": final_decoder_outputs, "final_decoder_state": final_decoder_state, "final_rnn_outputs": final_rnn_outputs, "final_rnn_state": final_rnn_state }) # Decoder only runs out to max_out; ensure values are identical # to dynamic_rnn, which also zeros out outputs and passes along state. self.assertAllClose( sess_results["final_decoder_outputs"].rnn_output, sess_results["final_rnn_outputs"][:, 0:max_out, :]) self.assertAllClose(sess_results["final_decoder_state"], sess_results["final_rnn_state"])
def rnn_logit_fn(features, mode): """Recurrent Neural Network logit_fn. Args: features: This is the first item returned from the `input_fn` passed to `train`, `evaluate`, and `predict`. This should be a single `Tensor` or `dict` of same. mode: Optional. Specifies if this training, evaluation or prediction. See `ModeKeys`. Returns: A tuple of `Tensor` objects representing the logits and the sequence length mask. """ with variable_scope.variable_scope( 'sequence_input_layer', values=tuple(six.itervalues(features)), partitioner=input_layer_partitioner): sequence_input, sequence_length = seq_fc.sequence_input_layer( features=features, feature_columns=sequence_feature_columns) summary.histogram('sequence_length', sequence_length) if context_feature_columns: context_input = feature_column_lib.input_layer( features=features, feature_columns=context_feature_columns) sequence_input = _concatenate_context_input( sequence_input, context_input) cell = rnn_cell_fn(mode) # Ignore output state. rnn_outputs, _ = rnn.dynamic_rnn(cell=cell, inputs=sequence_input, sequence_length=sequence_length, dtype=dtypes.float32, time_major=False) if not return_sequences: rnn_outputs = _select_last_activations(rnn_outputs, sequence_length) with variable_scope.variable_scope('logits', values=(rnn_outputs, )): logits = core_layers.dense( rnn_outputs, units=output_units, activation=None, kernel_initializer=init_ops.glorot_uniform_initializer()) sequence_length_mask = array_ops.sequence_mask(sequence_length) return logits, sequence_length_mask
def run_lstm_mnist(lstm_cell=BasicLSTMCell, hidden_size=32, batch_size=256, steps=1000): mnist = input_data.read_data_sets('MNIST_data', one_hot=True) learning_rate = 0.001 file_logger = FileLogger('log.tsv', ['step', 'training_loss', 'training_accuracy', 'testing_loss', 'testing_accuracy']) x = tf.placeholder('float32', [batch_size, 784, 2 if lstm_cell == PhasedLSTMCell else 1]) y_ = tf.placeholder('float32', [batch_size, 10]) initial_states = (tf.random_normal([batch_size, hidden_size], stddev=0.1), tf.random_normal([batch_size, hidden_size], stddev=0.1)) outputs, _ = dynamic_rnn(lstm_cell(hidden_size), x, initial_state=initial_states, dtype=tf.float32) rnn_out = tf.squeeze(outputs[:, -1, :]) fc0_w = create_weight_variable('fc0_w', [hidden_size, 10]) fc0_b = create_bias_variable('fc0_b', [10]) y = tf.matmul(rnn_out, fc0_w) + fc0_b cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_)) grad_update = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y_, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) sess = tf.Session(config=tf.ConfigProto(log_device_placement=False)) sess.run(tf.global_variables_initializer()) def transform_x(_x_): if lstm_cell == PhasedLSTMCell: t = np.reshape(np.tile(np.array(range(784)), (batch_size, 1)), (batch_size, 784)) return np.squeeze(np.stack([_x_, t], axis=2)) t_x = np.expand_dims(_x_, axis=2) return t_x test_images = mnist.test.images[0:256] test_labels = mnist.test.labels[0:256] for i in range(steps): batch = mnist.train.next_batch(batch_size) st = time() tr_feed_dict = {x: transform_x(batch[0]), y_: batch[1]} tr_loss, tr_acc, _ = sess.run([cross_entropy, accuracy, grad_update], feed_dict=tr_feed_dict) te_feed_dict = {x: transform_x(test_images), y_: test_labels} te_loss, te_acc = sess.run([cross_entropy, accuracy], feed_dict=te_feed_dict) print('Forward-Backward pass took {0:.2f}s to complete.'.format(time() - st)) file_logger.write([i, tr_loss, tr_acc, te_loss, te_acc]) file_logger.close()
def _tfGRU(x, initial_state, y): gru_cell = rnn_cell.GRUCell( num_hidden, name='gru_cell', kernel_initializer=init_ops.zeros_initializer(dtype=dataType), bias_initializer=init_ops.zeros_initializer(dtype=dataType)) outputs, _ = rnn.dynamic_rnn(gru_cell, x, dtype=dataType, initial_state=initial_state, time_major=True) softmax = nn.softmax_cross_entropy_with_logits_v2( logits=outputs[-1], labels=array_ops.stop_gradient(y)) loss = math_ops.reduce_mean(softmax) train = gradient_descent.GradientDescentOptimizer(lr).minimize(loss) return [loss, train]
def time_gru_net(self, hidden_units, input_data, input_length, type, scope='gru'): cell = self.build_time_aware_gru_cell_time(hidden_units) # cell = self.build_cell(hidden_units) self.input_length = tf.reshape(input_length, [-1]) outputs, _ = dynamic_rnn(cell, inputs=input_data, sequence_length=self.input_length, dtype=tf.float32, scope=scope) return outputs
def apply(self, is_train, x, mask=None): state = dynamic_rnn(self.cell_spec(is_train), x, mask, dtype=tf.float32)[1] if isinstance(self.output, int): return state[self.output] else: if self.output is None: if not isinstance(state, tf.Tensor): raise ValueError() return state for i, x in enumerate(state._fields): if x == self.output: return state[i] raise ValueError()
def testNoneDimsWithDynamicRNN(self): with self.test_session(use_gpu=self._use_gpu, graph=ops.Graph()) as sess: batch_size = 4 cell_size = 5 input_size = 6 num_steps = 7 cell = gru_ops.GRUBlockCell(cell_size) x = array_ops.placeholder(dtypes.float32, shape=(None, None, input_size)) _, output = rnn.dynamic_rnn( cell, x, time_major=True, dtype=dtypes.float32) sess.run(variables.global_variables_initializer()) feed = {} feed[x] = np.random.randn(num_steps, batch_size, input_size) sess.run(output, feed)
def dynamic_lstm_model_fn(batch_size, state_size, max_steps): # We make inputs and sequence_length constant so that multiple session.run # calls produce the same result. inputs = constant_op.constant( np.random.rand(batch_size, max_steps, state_size), dtype=dtypes.float32) sequence_length = constant_op.constant( np.random.randint(0, size=[batch_size], high=max_steps + 1), dtype=dtypes.int32) cell = rnn_cell.BasicLSTMCell(state_size) initial_state = cell.zero_state(batch_size, dtypes.float32) return inputs, rnn.dynamic_rnn( cell, inputs, sequence_length=sequence_length, initial_state=initial_state)