def bidirectional_rnn(cell_fw, cell_bw, inputs, initial_state_fw=None, initial_state_bw=None, dtype=None, sequence_length=None, scope=None): name = scope or "BiRNN" # Forward direction with vs.variable_scope(name + "_FW", initializer=tf.constant_initializer(0.005)): _, state_fw = rnn_return_states(cell_fw, inputs, initial_state_fw, dtype, sequence_length) # Backward direction with vs.variable_scope(name + "_BW", initializer=tf.constant_initializer(0.005)): _, tmp = rnn_return_states(cell_bw, _reverse_seq(inputs, sequence_length), initial_state_bw, dtype, sequence_length) state_bw = _reverse_seq(tmp, sequence_length) # Concat each of the forward/backward outputs states = [ array_ops.concat(1, [fw, bw]) for fw, bw in zip(state_fw, state_bw) ] return states
def _get_last_state(max_norm, char_lens, outputs): # A bit of a hack to get the last real output state. outputs = _reverse_seq(outputs, tf.cast(char_lens, dtype=tf.int64)) output = outputs[0] if max_norm: output = clip_by_norm(output, max_norm) return output
def bidirectional_rnn_tied(cell_fw, cell_bw, inputs, initial_state_fw=None, initial_state_bw=None, dtype=None, sequence_length=None, scope=None): name = scope or "BiRNN" with variable_scope.variable_scope(name) as fw_scope: # Forward direction output_fw, output_state_fw = tf.nn.rnn(cell_fw, inputs, initial_state_fw, dtype, sequence_length, scope=fw_scope) with variable_scope.variable_scope(name, reuse=True) as bw_scope: # Backward direction tmp, output_state_bw = tf.nn.rnn(cell_bw, _reverse_seq(inputs, sequence_length), initial_state_bw, dtype, sequence_length, scope=bw_scope) output_bw = _reverse_seq(tmp, sequence_length) # Concat each of the forward/backward outputs outputs = [array_ops.concat(1, [fw, bw]) for fw, bw in zip(output_fw, output_bw)] return (outputs, output_state_fw, output_state_bw)
def bidirectional_rnn(cell_fw, cell_bw, inputs, initial_state_fw=None, initial_state_bw=None, dtype=None, sequence_length=None, scope=None): """Creates a bidirectional recurrent neural network. Similar to the unidirectional case above (rnn) but takes input and builds independent forward and backward RNNs with the final forward and backward outputs depth-concatenated, such that the output will have the format [time][batch][cell_fw.output_size + cell_bw.output_size]. The input_size of forward and backward cell must match. The initial state for both directions is zero by default (but can be set optionally) and no intermediate states are ever returned -- the network is fully unrolled for the given (passed in) length(s) of the sequence(s) or completely unrolled if length(s) is not given. Args: cell_fw: An instance of RNNCell, to be used for forward direction. cell_bw: An instance of RNNCell, to be used for backward direction. inputs: A length T list of inputs, each a tensor of shape [batch_size, cell.input_size]. initial_state_fw: (optional) An initial state for the forward RNN. This must be a tensor of appropriate type and shape [batch_size x cell.state_size]. initial_state_bw: (optional) Same as for initial_state_fw. dtype: (optional) The data type for the initial state. Required if either of the initial states are not provided. sequence_length: (optional) An int32/int64 vector, size [batch_size], containing the actual lengths for each of the sequences. scope: VariableScope for the created subgraph; defaults to "BiRNN" Returns: A set of output `Tensors` where: outputs is a length T list of outputs (one for each input), which are depth-concatenated forward and backward outputs Raises: TypeError: If "cell_fw" or "cell_bw" is not an instance of RNNCell. ValueError: If inputs is None or an empty list. """ if not isinstance(cell_fw, tf.nn.rnn_cell.RNNCell): raise TypeError("cell_fw must be an instance of RNNCell") if not isinstance(cell_bw, tf.nn.rnn_cell.RNNCell): raise TypeError("cell_bw must be an instance of RNNCell") if not isinstance(inputs, list): raise TypeError("inputs must be a list") if not inputs: raise ValueError("inputs must not be empty") name = scope or "BiRNN" # Forward direction with tf.variable_scope(name + "_FW") as fw_scope: output_fw, _ = tf.nn.rnn(cell_fw, inputs, initial_state_fw, dtype, sequence_length, scope=fw_scope) # Backward direction with tf.variable_scope(name + "_BW") as bw_scope: tmp,_,c1h1 = custom_rnn(cell_bw, _reverse_seq(inputs, sequence_length), initial_state_bw, dtype, sequence_length, scope=bw_scope) output_bw = _reverse_seq(tmp, sequence_length) # Concat each of the forward/backward outputs outputs = [tf.concat(1, [fw, bw]) for fw, bw in zip(output_fw, output_bw)] # get last memory state of bw cell return outputs,c1h1
def bidirectional_rnn(cell_fw, cell_bw, inputs, initial_state_fw=None, initial_state_bw=None, dtype=None, sequence_length=None, scope=None): """Creates a bidirectional recurrent neural network. Similar to the unidirectional case above (rnn) but takes input and builds independent forward and backward RNNs with the final forward and backward outputs depth-concatenated, such that the output will have the format [time][batch][cell_fw.output_size + cell_bw.output_size]. The input_size of forward and backward cell must match. The initial state for both directions is zero by default (but can be set optionally) and no intermediate states are ever returned -- the network is fully unrolled for the given (passed in) length(s) of the sequence(s) or completely unrolled if length(s) is not given. Args: cell_fw: An instance of RNNCell, to be used for forward direction. cell_bw: An instance of RNNCell, to be used for backward direction. inputs: A length T list of inputs, each a tensor of shape [batch_size, cell.input_size]. initial_state_fw: (optional) An initial state for the forward RNN. This must be a tensor of appropriate type and shape [batch_size x cell.state_size]. initial_state_bw: (optional) Same as for initial_state_fw. dtype: (optional) The data type for the initial state. Required if either of the initial states are not provided. sequence_length: (optional) An int32/int64 vector, size [batch_size], containing the actual lengths for each of the sequences. scope: VariableScope for the created subgraph; defaults to "BiRNN" Returns: A set of output `Tensors` where: outputs is a length T list of outputs (one for each input), which are depth-concatenated forward and backward outputs Raises: TypeError: If "cell_fw" or "cell_bw" is not an instance of RNNCell. ValueError: If inputs is None or an empty list. """ if not isinstance(cell_fw, tf.nn.rnn_cell.RNNCell): raise TypeError("cell_fw must be an instance of RNNCell") if not isinstance(cell_bw, tf.nn.rnn_cell.RNNCell): raise TypeError("cell_bw must be an instance of RNNCell") if not isinstance(inputs, list): raise TypeError("inputs must be a list") if not inputs: raise ValueError("inputs must not be empty") name = scope or "BiRNN" # Forward direction with tf.variable_scope(name + "_FW") as fw_scope: output_fw, _ = tf.nn.rnn(cell_fw, inputs, initial_state_fw, dtype, sequence_length, scope=fw_scope) # Backward direction with tf.variable_scope(name + "_BW") as bw_scope: tmp, _, c1h1 = custom_rnn(cell_bw, _reverse_seq(inputs, sequence_length), initial_state_bw, dtype, sequence_length, scope=bw_scope) output_bw = _reverse_seq(tmp, sequence_length) # Concat each of the forward/backward outputs outputs = [tf.concat(1, [fw, bw]) for fw, bw in zip(output_fw, output_bw)] # get last memory state of bw cell return outputs, c1h1