def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, dtype=None, parallel_iterations=None, swap_memory=False, time_major=True, scope=None): """Creates a recurrent neural network specified by RNNCell `cell`. Performs fully dynamic unrolling of `inputs`. Example: ```python # create a BasicRNNCell rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size) # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size] # defining initial state initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32) # 'state' is a tensor of shape [batch_size, cell_state_size] outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data, initial_state=initial_state, dtype=tf.float32) ``` ```python # create 2 LSTMCells rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]] # create a RNN cell composed sequentially of a number of RNNCells multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers) # 'outputs' is a tensor of shape [batch_size, max_time, 256] # 'state' is a N-tuple where N is the number of LSTMCells containing a # tf.contrib.rnn.LSTMStateTuple for each cell outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell, inputs=data, dtype=tf.float32) ``` Args: cell: An instance of RNNCell. inputs: The RNN inputs. If `time_major == False` (default), this must be a `Tensor` of shape: `[batch_size, max_time, ...]`, or a nested tuple of such elements. If `time_major == True`, this must be a `Tensor` of shape: `[max_time, batch_size, ...]`, or a nested tuple of such elements. This may also be a (possibly nested) tuple of Tensors satisfying this property. The first two dimensions must match across all the inputs, but otherwise the ranks and other shape components may differ. In this case, input to `cell` at each time-step will replicate the structure of these tuples, except for the time dimension (from which the time is taken). The input to `cell` at each time step will be a `Tensor` or (possibly nested) tuple of Tensors each with dimensions `[batch_size, ...]`. sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. Used to copy-through state and zero-out outputs when past a batch element's sequence length. So it's more for performance than correctness. initial_state: (optional) An initial state for the RNN. If `cell.state_size` is an integer, this must be a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`. If `cell.state_size` is a tuple, this should be a tuple of tensors having shapes `[batch_size, s] for s in cell.state_size`. dtype: (optional) The data type for the initial state and expected output. Required if initial_state is not provided or RNN state has a heterogeneous dtype. parallel_iterations: (Default: 32). The number of iterations to run in parallel. Those operations which do not have any temporal dependency and can be run in parallel, will be. This parameter trades off time for space. Values >> 1 use more memory but take less time, while smaller values use less memory but computations take longer. swap_memory: Transparently swap the tensors produced in forward inference but needed for back prop from GPU to CPU. This allows training RNNs which would typically not fit on a single GPU, with very minimal (or no) performance penalty. time_major: The shape format of the `inputs` and `outputs` Tensors. If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. Using `time_major = True` is a bit more efficient because it avoids transposes at the beginning and end of the RNN calculation. However, most TensorFlow data is batch-major, so by default this function accepts input and emits output in batch-major form. scope: VariableScope for the created subgraph; defaults to "rnn". Returns: A pair (outputs, state) where: outputs: The RNN output `Tensor`. If time_major == False (default), this will be a `Tensor` shaped: `[batch_size, max_time, cell.output_size]`. If time_major == True, this will be a `Tensor` shaped: `[max_time, batch_size, cell.output_size]`. Note, if `cell.output_size` is a (possibly nested) tuple of integers or `TensorShape` objects, then `outputs` will be a tuple having the same structure as `cell.output_size`, containing Tensors having shapes corresponding to the shape data in `cell.output_size`. state: The final state. If `cell.state_size` is an int, this will be shaped `[batch_size, cell.state_size]`. If it is a `TensorShape`, this will be shaped `[batch_size] + cell.state_size`. If it is a (possibly nested) tuple of ints or `TensorShape`, this will be a tuple having the corresponding shapes. If cells are `LSTMCells` `state` will be a tuple containing a `LSTMStateTuple` for each cell. Raises: TypeError: If `cell` is not an instance of RNNCell. ValueError: If inputs is None or an empty list. RuntimeError: If not using control flow v2. """ # Currently only support time_major == True case. assert time_major # TODO(b/123051275): We need to check if the cells are TfLiteLSTMCells or # TfLiteRNNCells. rnn_cell_impl.assert_like_rnncell("cell", cell) if not control_flow_util.ENABLE_CONTROL_FLOW_V2: raise RuntimeError("OpHint dynamic rnn only supports control flow v2.") parent_first_child_input = [{ "parent_ophint_input_index": 0, "first_child_ophint_input_index": 0 }] parent_last_child_output = [{ "parent_output_index": 0, # For LstmCell, the index is 2. # For RnnCell, the index is 1. # So we use -1 meaning it's the last one. "child_output_index": -1 }] internal_children_input_output = [{ "child_input_index": 0, # For LstmCell, the index is 2. # For RnnCell, the index is 1. # So we use -1 meaning it's the last one. "child_output_index": -1 }] inputs_outputs_mappings = { "parent_first_child_input": parent_first_child_input, "parent_last_child_output": parent_last_child_output, "internal_children_input_output": internal_children_input_output } tflite_wrapper = op_hint.OpHint( "TfLiteDynamicRnn", level=2, children_inputs_mappings=inputs_outputs_mappings) with vs.variable_scope(scope or "rnn") as varscope: # Create a new scope in which the caching device is either # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. if _should_cache(): if varscope.caching_device is None: varscope.set_caching_device(lambda op: op.device) inputs = tflite_wrapper.add_input(inputs, name="input", index_override=0) # By default, time_major==False and inputs are batch-major: shaped # [batch, time, depth] # For internal calculations, we transpose to [time, batch, depth] flat_input = nest.flatten(inputs) if not time_major: # (batch, time, depth) => (time, batch, depth) flat_input = [ ops.convert_to_tensor(input_) for input_ in flat_input ] flat_input = tuple( _transpose_batch_time(input_) for input_ in flat_input) parallel_iterations = parallel_iterations or 32 if sequence_length is not None: sequence_length = math_ops.cast(sequence_length, dtypes.int32) if sequence_length.shape.rank not in (None, 1): raise ValueError( "sequence_length must be a vector of length batch_size, " "but saw shape: %s" % sequence_length.shape) sequence_length = array_ops.identity( # Just to find it in the graph. sequence_length, name="sequence_length") batch_size = _best_effort_input_batch_size(flat_input) if initial_state is not None: state = initial_state else: if not dtype: raise ValueError( "If there is no initial_state, you must give a dtype.") if getattr(cell, "get_initial_state", None) is not None: state = cell.get_initial_state(inputs=None, batch_size=batch_size, dtype=dtype) else: state = cell.zero_state(batch_size, dtype) def _assert_has_shape(x, shape): x_shape = array_ops.shape(x) packed_shape = array_ops.stack(shape) return control_flow_ops.Assert( math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)), [ "Expected shape for Tensor %s is " % x.name, packed_shape, " but saw shape: ", x_shape ]) if not context.executing_eagerly() and sequence_length is not None: # Perform some shape validation with ops.control_dependencies( [_assert_has_shape(sequence_length, [batch_size])]): sequence_length = array_ops.identity(sequence_length, name="CheckSeqLen") inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input) outputs, final_state = _dynamic_rnn_loop( cell, inputs, state, parallel_iterations=parallel_iterations, swap_memory=swap_memory, sequence_length=sequence_length, dtype=dtype) # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth]. # If we are performing batch-major calculations, transpose output back # to shape [batch, time, depth] if not time_major: # (time, batch, depth) => (batch, time, depth) outputs = nest.map_structure(_transpose_batch_time, outputs) outputs = tflite_wrapper.add_output(outputs, name="outputs") return outputs, final_state
def changing_ndim_rnn_tf(step_function, inputs, initial_states, go_backwards, mask, constants, unroll, input_length, eliminate_mask_dims): '''Iterates over the time dimension of a tensor. # Arguments inputs: tensor of temporal data of shape (samples, time, ...) (at least 3D). step_function: Parameters: input: tensor with shape (samples, ...) (no time dimension), representing input for the batch of samples at a certain time step. states: list of tensors. Returns: output: tensor with shape (samples, output_dim) (no time dimension), new_states: list of tensors, same length and shapes as 'states'. The first state in the list must be the output tensor at the previous timestep. initial_states: tensor with shape (samples, output_dim) (no time dimension), containing the initial values for the states used in the step function. go_backwards: boolean. If True, do the iteration over the time dimension in reverse order. mask: binary tensor with shape (samples, time, 1), with a zero for every element that is masked. constants: a list of constant values passed at each step. unroll: with TensorFlow the RNN is always unrolled, but with Theano you can use this boolean flag to unroll the RNN. input_length: not relevant in the TensorFlow implementation. Must be specified if using unrolling with Theano. # Returns A tuple (last_output, outputs, new_states). last_output: the latest output of the rnn, of shape (samples, ...) outputs: tensor with shape (samples, time, ...) where each entry outputs[s, t] is the output of the step function at time t for sample s. new_states: list of tensors, latest states returned by the step function, of shape (samples, ...). ''' import tensorflow as tf ndim = len(inputs.get_shape()) assert ndim >= 3, 'Input should be at least 3D.' axes = [1, 0] + list(range(2, ndim)) inputs = tf.transpose(inputs, (axes)) if constants is None: constants = [] if unroll: if not inputs.get_shape()[0]: raise Exception('Unrolling requires a fixed number of timesteps.') states = initial_states successive_states = [] successive_outputs = [] input_list = tf.unpack(inputs) if go_backwards: input_list.reverse() if mask is not None: # Transpose not supported by bool tensor types, hence round-trip to uint8. mask = tf.cast(mask, tf.uint8) if len(mask.get_shape()) == ndim - 1: mask = K.expand_dims(mask) # Reshaping mask to make timesteps the first dimension. mask = tf.cast(tf.transpose(mask, axes), tf.bool) mask_list = tf.unpack(mask) if go_backwards: mask_list.reverse() # Iterating over timesteps. for input, mask_t in zip(input_list, mask_list): # Changing ndim modification: Pass the mask to the step function as a constant. output, new_states = step_function( input, states + constants + [mask_t]) # tf.select needs its condition tensor to be the same shape as its two # result tensors, but in our case the condition (mask) tensor is # (nsamples, 1), and A and B are (nsamples, ndimensions). So we need to # broadcast the mask to match the shape of A and B. That's what the # tile call does, is just repeat the mask along its second dimension # ndimensions times. output_mask_t = tf.tile( mask_t, tf.pack(([1] * (ndim - 2)) + [tf.shape(output)[1]])) if len(successive_outputs) == 0: prev_output = K.zeros_like(output) else: prev_output = successive_outputs[-1] # Changing ndim modification: Define output mask with appropriate dims eliminated. if eliminate_mask_dims is not None: output_mask_t = tf.cast( K.any(output_mask_t, axis=eliminate_mask_dims), tf.bool) else: output_mask_t = tf.cast(output_mask_t, tf.bool) output = tf.select(output_mask_t, output, prev_output) return_states = [] for state, new_state in zip(states, new_states): # (see earlier comment for tile explanation) state_mask_t = tf.tile( mask_t, tf.pack(([1] * (ndim - 2)) + [tf.shape(new_state)[1]])) # Changing ndim modification: Define output mask with appropriate dims eliminated. if eliminate_mask_dims is not None: state_mask_t = tf.cast( K.any(state_mask_t, axis=eliminate_mask_dims), tf.bool) else: state_mask_t = tf.cast(state_mask_t, tf.bool) return_states.append( tf.select(state_mask_t, new_state, state)) states = return_states successive_outputs.append(output) successive_states.append(states) last_output = successive_outputs[-1] new_states = successive_states[-1] outputs = tf.pack(successive_outputs) else: for input in input_list: output, states = step_function(input, states + constants + [None]) # None for mask successive_outputs.append(output) successive_states.append(states) last_output = successive_outputs[-1] new_states = successive_states[-1] outputs = tf.pack(successive_outputs) else: from tensorflow.python.ops.rnn import _dynamic_rnn_loop if go_backwards: inputs = tf.reverse(inputs, [True] + [False] * (ndim - 1)) states = initial_states nb_states = len(states) if nb_states == 0: # use dummy state, otherwise _dynamic_rnn_loop breaks state = inputs[:, 0, :] state_size = state.get_shape()[-1] else: state_size = int(states[0].get_shape()[-1]) if nb_states == 1: state = states[0] else: state = tf.concat(1, states) if mask is not None: if len(initial_states) == 0: raise ValueError('No initial states provided! ' 'When using masking in an RNN, you should ' 'provide initial states ' '(and your step function should return ' 'as its first state at time `t` ' 'the output at time `t-1`).') if go_backwards: mask = tf.reverse(mask, [True] + [False] * (ndim - 2)) # Transpose not supported by bool tensor types, hence round-trip to uint8. mask = tf.cast(mask, tf.uint8) if len(mask.get_shape()) == ndim - 1: mask = K.expand_dims(mask) mask = tf.transpose(mask, axes) # Concatenate at the last dim. inputs = tf.concat(ndim - 1, [tf.cast(mask, inputs.dtype), inputs]) def _step(input, state): if nb_states > 1: states = [] for i in range(nb_states): states.append(state[:, i * state_size:(i + 1) * state_size]) else: states = [state] # The time dimension is not present here. step_ndim = ndim - 1 # Permuting only to take out the mask. permuted_input = K.permute_dimensions( input, (step_ndim - 1, ) + tuple(range(step_ndim - 1))) mask_t = K.expand_dims(permuted_input[0]) permuted_input = permuted_input[1:] input = K.permute_dimensions( permuted_input, tuple(range(1, step_ndim)) + (0, )) # changing ndim fix: eliminate necessary dims after selecting the mask from the input. if eliminate_mask_dims is not None: output_mask_t = K.sum(mask_t, axis=eliminate_mask_dims) mask_t = tf.cast(mask_t, tf.bool) output_mask_t = tf.cast(output_mask_t, tf.bool) output, new_states = step_function( input, states + constants + [mask_t]) tiled_output_mask_t = tf.tile( output_mask_t, tf.pack([1, tf.shape(output)[1]])) output = tf.select(tiled_output_mask_t, output, states[0]) return_states = [] for state, new_state in zip(states, new_states): tiled_state_mask_t = tf.tile( output_mask_t, tf.pack([1, tf.shape(state)[1]])) return_states.append( tf.select(tiled_state_mask_t, new_state, state)) if len(return_states) == 1: new_state = return_states[0] else: new_state = tf.concat(1, return_states) return output, new_state else: def _step(input, state): if nb_states > 1: states = [] for i in range(nb_states): states.append(state[:, i * state_size:(i + 1) * state_size]) elif nb_states == 1: states = [state] else: states = [] output, new_states = step_function(input, states + constants + [None]) # None for mask if len(new_states) > 1: new_state = tf.concat(1, new_states) elif len(new_states) == 1: new_state = new_states[0] else: # return dummy state, otherwise _dynamic_rnn_loop breaks new_state = output return output, new_state _step.state_size = state_size * nb_states # recover output size by calling _step on the first input slice_begin = tf.pack([0] * ndim) slice_size = tf.pack([1] + [-1] * (ndim - 1)) first_input = tf.slice(inputs, slice_begin, slice_size) first_input = tf.squeeze(first_input, [0]) _step.output_size = int(_step(first_input, state)[0].get_shape()[-1]) (outputs, final_state) = _dynamic_rnn_loop(_step, inputs, state, parallel_iterations=32, swap_memory=True, sequence_length=None) if nb_states > 1: new_states = [] for i in range(nb_states): new_states.append(final_state[:, i * state_size:(i + 1) * state_size]) elif nb_states == 1: new_states = [final_state] else: new_states = [] outputs_ndim = len(outputs.get_shape()) # all this circus is to recover the last vector in the sequence. slice_begin = tf.pack([tf.shape(outputs)[0] - 1] + [0] * (outputs_ndim - 1)) slice_size = tf.pack([1] + [-1] * (outputs_ndim - 1)) last_output = tf.slice(outputs, slice_begin, slice_size) last_output = tf.squeeze(last_output, [0]) axes = [1, 0] + list(range(2, len(outputs.get_shape()))) outputs = tf.transpose(outputs, axes) return last_output, outputs, new_states
def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, dtype=None, parallel_iterations=None, swap_memory=False, time_major=True, scope=None): """Creates a recurrent neural network specified by RNNCell `cell`. Performs fully dynamic unrolling of `inputs`. Example: ```python # create a BasicRNNCell rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size) # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size] # defining initial state initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32) # 'state' is a tensor of shape [batch_size, cell_state_size] outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data, initial_state=initial_state, dtype=tf.float32) ``` ```python # create 2 LSTMCells rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]] # create a RNN cell composed sequentially of a number of RNNCells multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers) # 'outputs' is a tensor of shape [batch_size, max_time, 256] # 'state' is a N-tuple where N is the number of LSTMCells containing a # tf.contrib.rnn.LSTMStateTuple for each cell outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell, inputs=data, dtype=tf.float32) ``` Args: cell: An instance of RNNCell. inputs: The RNN inputs. If `time_major == False` (default), this must be a `Tensor` of shape: `[batch_size, max_time, ...]`, or a nested tuple of such elements. If `time_major == True`, this must be a `Tensor` of shape: `[max_time, batch_size, ...]`, or a nested tuple of such elements. This may also be a (possibly nested) tuple of Tensors satisfying this property. The first two dimensions must match across all the inputs, but otherwise the ranks and other shape components may differ. In this case, input to `cell` at each time-step will replicate the structure of these tuples, except for the time dimension (from which the time is taken). The input to `cell` at each time step will be a `Tensor` or (possibly nested) tuple of Tensors each with dimensions `[batch_size, ...]`. sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. Used to copy-through state and zero-out outputs when past a batch element's sequence length. So it's more for performance than correctness. initial_state: (optional) An initial state for the RNN. If `cell.state_size` is an integer, this must be a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`. If `cell.state_size` is a tuple, this should be a tuple of tensors having shapes `[batch_size, s] for s in cell.state_size`. dtype: (optional) The data type for the initial state and expected output. Required if initial_state is not provided or RNN state has a heterogeneous dtype. parallel_iterations: (Default: 32). The number of iterations to run in parallel. Those operations which do not have any temporal dependency and can be run in parallel, will be. This parameter trades off time for space. Values >> 1 use more memory but take less time, while smaller values use less memory but computations take longer. swap_memory: Transparently swap the tensors produced in forward inference but needed for back prop from GPU to CPU. This allows training RNNs which would typically not fit on a single GPU, with very minimal (or no) performance penalty. time_major: The shape format of the `inputs` and `outputs` Tensors. If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. Using `time_major = True` is a bit more efficient because it avoids transposes at the beginning and end of the RNN calculation. However, most TensorFlow data is batch-major, so by default this function accepts input and emits output in batch-major form. scope: VariableScope for the created subgraph; defaults to "rnn". Returns: A pair (outputs, state) where: outputs: The RNN output `Tensor`. If time_major == False (default), this will be a `Tensor` shaped: `[batch_size, max_time, cell.output_size]`. If time_major == True, this will be a `Tensor` shaped: `[max_time, batch_size, cell.output_size]`. Note, if `cell.output_size` is a (possibly nested) tuple of integers or `TensorShape` objects, then `outputs` will be a tuple having the same structure as `cell.output_size`, containing Tensors having shapes corresponding to the shape data in `cell.output_size`. state: The final state. If `cell.state_size` is an int, this will be shaped `[batch_size, cell.state_size]`. If it is a `TensorShape`, this will be shaped `[batch_size] + cell.state_size`. If it is a (possibly nested) tuple of ints or `TensorShape`, this will be a tuple having the corresponding shapes. If cells are `LSTMCells` `state` will be a tuple containing a `LSTMStateTuple` for each cell. Raises: TypeError: If `cell` is not an instance of RNNCell. ValueError: If inputs is None or an empty list. RuntimeError: If not using control flow v2. """ # Currently only support time_major == True case. assert time_major # TODO(b/123051275): We need to check if the cells are TfLiteLSTMCells or # TfLiteRNNCells. rnn_cell_impl.assert_like_rnncell("cell", cell) if not control_flow_util.ENABLE_CONTROL_FLOW_V2: raise RuntimeError("OpHint dynamic rnn only supports control flow v2.") parent_first_child_input = [{ "parent_ophint_input_index": 0, "first_child_ophint_input_index": 0 }] parent_last_child_output = [{ "parent_output_index": 0, # For LstmCell, the index is 2. # For RnnCell, the index is 1. # So we use -1 meaning it's the last one. "child_output_index": -1 }] internal_children_input_output = [{ "child_input_index": 0, # For LstmCell, the index is 2. # For RnnCell, the index is 1. # So we use -1 meaning it's the last one. "child_output_index": -1 }] inputs_outputs_mappings = { "parent_first_child_input": parent_first_child_input, "parent_last_child_output": parent_last_child_output, "internal_children_input_output": internal_children_input_output } tflite_wrapper = op_hint.OpHint( "TfLiteDynamicRnn", level=2, children_inputs_mappings=inputs_outputs_mappings) with vs.variable_scope(scope or "rnn") as varscope: # Create a new scope in which the caching device is either # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. if _should_cache(): if varscope.caching_device is None: varscope.set_caching_device(lambda op: op.device) inputs = tflite_wrapper.add_input(inputs, name="input", index_override=0) # By default, time_major==False and inputs are batch-major: shaped # [batch, time, depth] # For internal calculations, we transpose to [time, batch, depth] flat_input = nest.flatten(inputs) if not time_major: # (batch, time, depth) => (time, batch, depth) flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input] flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input) parallel_iterations = parallel_iterations or 32 if sequence_length is not None: sequence_length = math_ops.to_int32(sequence_length) if sequence_length.get_shape().rank not in (None, 1): raise ValueError( "sequence_length must be a vector of length batch_size, " "but saw shape: %s" % sequence_length.get_shape()) sequence_length = array_ops.identity( # Just to find it in the graph. sequence_length, name="sequence_length") batch_size = _best_effort_input_batch_size(flat_input) if initial_state is not None: state = initial_state else: if not dtype: raise ValueError("If there is no initial_state, you must give a dtype.") if getattr(cell, "get_initial_state", None) is not None: state = cell.get_initial_state( inputs=None, batch_size=batch_size, dtype=dtype) else: state = cell.zero_state(batch_size, dtype) def _assert_has_shape(x, shape): x_shape = array_ops.shape(x) packed_shape = array_ops.stack(shape) return control_flow_ops.Assert( math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)), [ "Expected shape for Tensor %s is " % x.name, packed_shape, " but saw shape: ", x_shape ]) if not context.executing_eagerly() and sequence_length is not None: # Perform some shape validation with ops.control_dependencies( [_assert_has_shape(sequence_length, [batch_size])]): sequence_length = array_ops.identity( sequence_length, name="CheckSeqLen") inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input) outputs, final_state = _dynamic_rnn_loop( cell, inputs, state, parallel_iterations=parallel_iterations, swap_memory=swap_memory, sequence_length=sequence_length, dtype=dtype) # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth]. # If we are performing batch-major calculations, transpose output back # to shape [batch, time, depth] if not time_major: # (time, batch, depth) => (batch, time, depth) outputs = nest.map_structure(_transpose_batch_time, outputs) outputs = tflite_wrapper.add_output(outputs, name="outputs") return outputs, final_state
def changing_ndim_rnn_tf(step_function, inputs, initial_states, go_backwards, mask, constants, unroll, input_length, eliminate_mask_dims): '''Iterates over the time dimension of a tensor. # Arguments inputs: tensor of temporal data of shape (samples, time, ...) (at least 3D). step_function: Parameters: input: tensor with shape (samples, ...) (no time dimension), representing input for the batch of samples at a certain time step. states: list of tensors. Returns: output: tensor with shape (samples, output_dim) (no time dimension), new_states: list of tensors, same length and shapes as 'states'. The first state in the list must be the output tensor at the previous timestep. initial_states: tensor with shape (samples, output_dim) (no time dimension), containing the initial values for the states used in the step function. go_backwards: boolean. If True, do the iteration over the time dimension in reverse order. mask: binary tensor with shape (samples, time, 1), with a zero for every element that is masked. constants: a list of constant values passed at each step. unroll: with TensorFlow the RNN is always unrolled, but with Theano you can use this boolean flag to unroll the RNN. input_length: not relevant in the TensorFlow implementation. Must be specified if using unrolling with Theano. # Returns A tuple (last_output, outputs, new_states). last_output: the latest output of the rnn, of shape (samples, ...) outputs: tensor with shape (samples, time, ...) where each entry outputs[s, t] is the output of the step function at time t for sample s. new_states: list of tensors, latest states returned by the step function, of shape (samples, ...). ''' import tensorflow as tf ndim = len(inputs.get_shape()) assert ndim >= 3, 'Input should be at least 3D.' axes = [1, 0] + list(range(2, ndim)) inputs = tf.transpose(inputs, (axes)) if constants is None: constants = [] if unroll: if not inputs.get_shape()[0]: raise Exception('Unrolling requires a fixed number of timesteps.') states = initial_states successive_states = [] successive_outputs = [] input_list = tf.unpack(inputs) if go_backwards: input_list.reverse() if mask is not None: # Transpose not supported by bool tensor types, hence round-trip to uint8. mask = tf.cast(mask, tf.uint8) if len(mask.get_shape()) == ndim - 1: mask = K.expand_dims(mask) # Reshaping mask to make timesteps the first dimension. mask = tf.cast(tf.transpose(mask, axes), tf.bool) mask_list = tf.unpack(mask) if go_backwards: mask_list.reverse() # Iterating over timesteps. for input, mask_t in zip(input_list, mask_list): # Changing ndim modification: Pass the mask to the step function as a constant. output, new_states = step_function(input, states + constants + [mask_t]) # tf.select needs its condition tensor to be the same shape as its two # result tensors, but in our case the condition (mask) tensor is # (nsamples, 1), and A and B are (nsamples, ndimensions). So we need to # broadcast the mask to match the shape of A and B. That's what the # tile call does, is just repeat the mask along its second dimension # ndimensions times. output_mask_t = tf.tile(mask_t, tf.pack(([1] * (ndim-2)) + [tf.shape(output)[1]])) if len(successive_outputs) == 0: prev_output = K.zeros_like(output) else: prev_output = successive_outputs[-1] # Changing ndim modification: Define output mask with appropriate dims eliminated. if eliminate_mask_dims is not None: output_mask_t = tf.cast(K.any(output_mask_t, axis=eliminate_mask_dims), tf.bool) else: output_mask_t = tf.cast(output_mask_t, tf.bool) output = tf.select(output_mask_t, output, prev_output) return_states = [] for state, new_state in zip(states, new_states): # (see earlier comment for tile explanation) state_mask_t = tf.tile(mask_t, tf.pack(([1] * (ndim-2)) + [tf.shape(new_state)[1]])) # Changing ndim modification: Define output mask with appropriate dims eliminated. if eliminate_mask_dims is not None: state_mask_t = tf.cast(K.any(state_mask_t, axis=eliminate_mask_dims), tf.bool) else: state_mask_t = tf.cast(state_mask_t, tf.bool) return_states.append(tf.select(state_mask_t, new_state, state)) states = return_states successive_outputs.append(output) successive_states.append(states) last_output = successive_outputs[-1] new_states = successive_states[-1] outputs = tf.pack(successive_outputs) else: for input in input_list: output, states = step_function(input, states + constants + [None]) # None for mask successive_outputs.append(output) successive_states.append(states) last_output = successive_outputs[-1] new_states = successive_states[-1] outputs = tf.pack(successive_outputs) else: from tensorflow.python.ops.rnn import _dynamic_rnn_loop if go_backwards: inputs = tf.reverse(inputs, [True] + [False] * (ndim - 1)) states = initial_states nb_states = len(states) if nb_states == 0: # use dummy state, otherwise _dynamic_rnn_loop breaks state = inputs[:, 0, :] state_size = state.get_shape()[-1] else: state_size = int(states[0].get_shape()[-1]) if nb_states == 1: state = states[0] else: state = tf.concat(1, states) if mask is not None: if len(initial_states) == 0: raise ValueError('No initial states provided! ' 'When using masking in an RNN, you should ' 'provide initial states ' '(and your step function should return ' 'as its first state at time `t` ' 'the output at time `t-1`).') if go_backwards: mask = tf.reverse(mask, [True] + [False] * (ndim - 2)) # Transpose not supported by bool tensor types, hence round-trip to uint8. mask = tf.cast(mask, tf.uint8) if len(mask.get_shape()) == ndim - 1: mask = K.expand_dims(mask) mask = tf.transpose(mask, axes) # Concatenate at the last dim. inputs = tf.concat(ndim-1, [tf.cast(mask, inputs.dtype), inputs]) def _step(input, state): if nb_states > 1: states = [] for i in range(nb_states): states.append(state[:, i * state_size: (i + 1) * state_size]) else: states = [state] # The time dimension is not present here. step_ndim = ndim - 1 # Permuting only to take out the mask. permuted_input = K.permute_dimensions(input, (step_ndim-1,) + tuple(range(step_ndim-1))) mask_t = K.expand_dims(permuted_input[0]) permuted_input = permuted_input[1:] input = K.permute_dimensions(permuted_input, tuple(range(1, step_ndim)) + (0,)) # changing ndim fix: eliminate necessary dims after selecting the mask from the input. if eliminate_mask_dims is not None: output_mask_t = K.sum(mask_t, axis=eliminate_mask_dims) mask_t = tf.cast(mask_t, tf.bool) output_mask_t = tf.cast(output_mask_t, tf.bool) output, new_states = step_function(input, states + constants + [mask_t]) tiled_output_mask_t = tf.tile(output_mask_t, tf.pack([1, tf.shape(output)[1]])) output = tf.select(tiled_output_mask_t, output, states[0]) return_states = [] for state, new_state in zip(states, new_states): tiled_state_mask_t = tf.tile(output_mask_t, tf.pack([1, tf.shape(state)[1]])) return_states.append(tf.select(tiled_state_mask_t, new_state, state)) if len(return_states) == 1: new_state = return_states[0] else: new_state = tf.concat(1, return_states) return output, new_state else: def _step(input, state): if nb_states > 1: states = [] for i in range(nb_states): states.append(state[:, i * state_size: (i + 1) * state_size]) elif nb_states == 1: states = [state] else: states = [] output, new_states = step_function(input, states + constants + [None]) # None for mask if len(new_states) > 1: new_state = tf.concat(1, new_states) elif len(new_states) == 1: new_state = new_states[0] else: # return dummy state, otherwise _dynamic_rnn_loop breaks new_state = output return output, new_state _step.state_size = state_size * nb_states # recover output size by calling _step on the first input slice_begin = tf.pack([0] * ndim) slice_size = tf.pack([1] + [-1] * (ndim - 1)) first_input = tf.slice(inputs, slice_begin, slice_size) first_input = tf.squeeze(first_input, [0]) _step.output_size = int(_step(first_input, state)[0].get_shape()[-1]) (outputs, final_state) = _dynamic_rnn_loop( _step, inputs, state, parallel_iterations=32, swap_memory=True, sequence_length=None) if nb_states > 1: new_states = [] for i in range(nb_states): new_states.append(final_state[:, i * state_size: (i + 1) * state_size]) elif nb_states == 1: new_states = [final_state] else: new_states = [] outputs_ndim = len(outputs.get_shape()) # all this circus is to recover the last vector in the sequence. slice_begin = tf.pack([tf.shape(outputs)[0] - 1] + [0] * (outputs_ndim - 1)) slice_size = tf.pack([1] + [-1] * (outputs_ndim - 1)) last_output = tf.slice(outputs, slice_begin, slice_size) last_output = tf.squeeze(last_output, [0]) axes = [1, 0] + list(range(2, len(outputs.get_shape()))) outputs = tf.transpose(outputs, axes) return last_output, outputs, new_states