def get_sequence_relevant_output(input, sequence_length): """Returns the last relevant output for a sequence.""" input_shape = get_shape(input) dynamic_input_shape = get_shape(input, dynamic=True) batch_size = input_shape[0] or dynamic_input_shape[0] max_length = input_shape[1] or dynamic_input_shape[1] dim_size = input_shape[2] or dynamic_input_shape[2] index = tf.range(batch_size) * max_length + (sequence_length - 1) flat = tf.reshape(input, [-1, dim_size]) relevant = tf.gather(flat, index) return relevant
def get_sequence_relevant_output(input, sequence_length): """Returns the last relevant output for a sequence.""" input_shape = get_shape(input) dynamic_input_shape = get_shape(input, dynamic=True) batch_size = input_shape[0] or dynamic_input_shape[0] max_length = input_shape[1] or dynamic_input_shape[1] dim_size = input_shape[2] or dynamic_input_shape[2] index = tf.range(batch_size) * max_length + (sequence_length - 1) flat = tf.reshape(input, [-1, dim_size]) relevant = tf.gather(flat, index) return relevant
def standardize(images): """Linearly scales `image` to have zero mean and unit norm. (A mirror to tf.image per_image_standardization) This op computes `(x - mean) / adjusted_stddev`, where `mean` is the average of all values in image, and `adjusted_stddev = max(stddev, 1.0/sqrt(image.NumElements()))`. `stddev` is the standard deviation of all values in `image`. It is capped away from zero to protect against division by 0 when handling uniform images. Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. Returns: The standardized image with same shape as `image`. Raises: ValueError: if the shape of 'image' is incompatible with this function. """ images_shape = get_shape(images) if images_shape > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if images_shape == 4: return tf.map_fn(lambda img: tf.image.per_image_standardization(img), images) return tf.image.per_image_standardization(images)
def rotate90(images, k=1, is_random=False, seed=None, name=None): """Rotate (randomly) images counter-clockwise by 90 degrees. (A mirror to tf.image rot90) Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. k: A scalar integer. The number of times the image is rotated by 90 degrees. is_random: `bool`, If True, adjust randomly. seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}. name: A name for this operation (optional). Returns: If `image` was 4-D, a 4-D float Tensor of shape `[batch, target_height, target_width, channels]` If `image` was 3-D, a 3-D float Tensor of shape `[target_height, target_width, channels] Raises: ValueError: if the shape of `image` not supported. """ if is_random: k = random_ops.random_shuffle([0, 1, 2, 3], seed=seed)[0] images_shape = get_shape(images) if images_shape > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if images_shape == 4: return tf.map_fn(lambda img: tf.image.rot90(img, k, name), images) return tf.image.rot90(images, k, name)
def decode(self, features, labels, decoder_fn, *args, **kwargs): """Decodes the incoming tensor if it's validates against the state size of the decoder. Otherwise, generates a random value. Args: features: `Tensor` labels: `dict` or `Tensor` decoder_fn: `function`. *args: **kwargs: """ incoming_shape = get_shape(features) if incoming_shape[1:] != self.state_size: raise ValueError('`incoming` tensor is incompatible with decoder function, ' 'expects a tensor with shape `{}`, ' 'received instead `{}`'.format(self.state_size, incoming_shape[1:])) # TODO: make decode capable of generating values directly, # TODO: basically accepting None incoming values. Should also specify a distribution. # shape = self._get_decoder_shape(incoming) # return decoder_fn(mode=self.mode, inputs=tf.random_normal(shape=shape)) if 'labels' in get_arguments(decoder_fn): kwargs['labels'] = labels x = decoder_fn(mode=self.mode, features=features, **kwargs) if not isinstance(x, DecoderSpec): raise ValueError('`decoder_fn` should return an DecoderSpec.') return x.output
def standardize(images): """Linearly scales `image` to have zero mean and unit norm. (A mirror to tf.image per_image_standardization) This op computes `(x - mean) / adjusted_stddev`, where `mean` is the average of all values in image, and `adjusted_stddev = max(stddev, 1.0/sqrt(image.NumElements()))`. `stddev` is the standard deviation of all values in `image`. It is capped away from zero to protect against division by 0 when handling uniform images. Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. Returns: The standardized image with same shape as `image`. Raises: ValueError: if the shape of 'image' is incompatible with this function. """ images_shape = get_shape(images) if len(images_shape) > 4: ValueError("'image' must have either 3 or 4 dimensions, " "received `{}`.".format(images_shape)) if len(images_shape) == 4: return tf.map_fn(lambda img: tf.image.per_image_standardization(img), images) return tf.image.per_image_standardization(images)
def decode(self, incoming, decoder_fn, *args, **kwargs): """Decodes the incoming tensor if it's validates against the state size of the decoder. Otherwise, generates a random value. Args: incoming: `Tensor` decoder_fn: `function`. *args: **kwargs: """ incoming_shape = get_shape(incoming) if incoming_shape[1:] != self.state_size: raise ValueError( '`incoming` tensor is incompatible with decoder function, ' 'expects a tensor with shape `{}`, ' 'received instead `{}`'.format(self.state_size, incoming_shape[1:])) # TODO: make decode capable of generating values directly, # TODO: basically accecpting None incoming values. Should also specify a distribution. # shape = self._get_decoder_shape(incoming) # return decoder_fn(mode=self.mode, inputs=tf.random_normal(shape=shape)) x = decoder_fn(mode=self.mode, inputs=incoming) if not isinstance(x, DecoderSpec): raise ValueError('`decoder_fn` should return an DecoderSpec.') return x.output
def transpose(images): """Transpose an image/images by swapping the first and second dimension. (A mirror to tf.image transpose_image) Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. Returns: If `image` was 4-D, a 4-D float Tensor of shape `[batch, target_height, target_width, channels]` If `image` was 3-D, a 3-D float Tensor of shape `[target_height, target_width, channels] Raises: ValueError: if the shape of `image` not supported. """ images_shape = get_shape(images) if len(images_shape) > 4: ValueError("'image' must have either 3 or 4 dimensions, " "received `{}`.".format(images_shape)) if len(images_shape) == 4: return tf.map_fn(lambda img: tf.image.transpose_image(img), images) return tf.image.transpose_image(images)
def _build(self, incoming, *args, **kwargs): """ Args: 2-D Tensor [samples, ids]. Returns: 3-D Tensor [samples, embedded_ids, features]. """ input_shape = get_shape(incoming) assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D' weights_init = getters.get_initializer(self.weights_init) self._w = variable('w', shape=[self.input_dim, self.output_dim], initializer=weights_init, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.cast(x=incoming, dtype=tf.int32) inference = tf.nn.embedding_lookup( params=self._w, ids=inference, validate_indices=self.validate_indices) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): """ Args: 2-D Tensor [samples, ids]. Returns: 3-D Tensor [samples, embedded_ids, features]. """ input_shape = get_shape(incoming) assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D' weights_init = getters.get_initializer(self.weights_init) self._w = variable('w', shape=[self.input_dim, self.output_dim], initializer=weights_init, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.cast(x=incoming, dtype=tf.int32) inference = tf.nn.embedding_lookup( params=self._w, ids=inference, validate_indices=self.validate_indices) # Embedding doesn't support masking, so we save sequence length prior to the lookup. # Expand dim to 3d. shape = [-1] + inference.get_shape().as_list()[1:3] + [1] inference.seq_length = retrieve_seq_length_op( tf.reshape(incoming, shape)) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def rotate90(images, k=1, is_random=False, seed=None, name=None): """Rotate (randomly) images counter-clockwise by 90 degrees. (A mirror to tf.image rot90) Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. k: A scalar integer. The number of times the image is rotated by 90 degrees. is_random: `bool`, If True, adjust randomly. seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}. name: A name for this operation (optional). Returns: If `image` was 4-D, a 4-D float Tensor of shape `[batch, target_height, target_width, channels]` If `image` was 3-D, a 3-D float Tensor of shape `[target_height, target_width, channels] Raises: ValueError: if the shape of `image` not supported. """ if is_random: k = random_ops.random_shuffle([0, 1, 2, 3], seed=seed)[0] images_shape = get_shape(images) if len(images_shape) > 4: ValueError("'image' must have either 3 or 4 dimensions, " "received `{}`.".format(images_shape)) if len(images_shape) == 4: return tf.map_fn(lambda img: tf.image.rot90(img, k, name), images) return tf.image.rot90(images, k, name)
def transpose(images): """Transpose an image/images by swapping the first and second dimension. (A mirror to tf.image transpose_image) Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. Returns: If `image` was 4-D, a 4-D float Tensor of shape `[batch, target_height, target_width, channels]` If `image` was 3-D, a 3-D float Tensor of shape `[target_height, target_width, channels] Raises: ValueError: if the shape of `image` not supported. """ images_shape = get_shape(images) if images_shape > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if images_shape == 4: return tf.map_fn(lambda img: tf.image.transpose_image(img), images) return tf.image.transpose_image(images)
def _build_loss(self, results, features, labels): """Creates the loss operation Returns: tuple `(losses, loss)`: `losses` are the per-batch losses. `loss` is a single scalar tensor to minimize. """ action = labels['action'] discount_reward = labels['discount_reward'] dist_values = labels['dist_values'] tangents = labels.get('tangents') theta = labels.get('theta') old_distribution = self._build_distribution(values=dist_values) log_probs = self._graph_results.distribution.log_prob(action) old_log_probs = old_distribution.log_prob(action) self._losses = tf.multiply(x=tf.exp(log_probs - old_log_probs), y=discount_reward) self._surrogate_loss = -tf.reduce_mean(self._losses, axis=0, name='surrogate_loss') entropy = self._graph_results.distribution.entropy() self._entropy_loss = tf.reduce_mean(entropy, name='entropy_loss') kl_divergence_value = kl_divergence(self._graph_results.distribution, old_distribution) self._kl_loss = tf.reduce_mean(kl_divergence_value, name='kl_loss') if self.is_continuous: dist_values_fixed = tf.stop_gradient( tf.concat(values=[self._graph_results.distribution.loc, self._graph_results.distribution.scale], axis=0)) else: dist_values_fixed = tf.stop_gradient(self._graph_results.distribution.logits) distribution_1_fixed = self._build_distribution(values=dist_values_fixed) kl_divergence_1_fixed = kl_divergence( distribution_1_fixed, self._graph_results.distribution) self._kl_loss_1_fixed = tf.reduce_mean(kl_divergence_1_fixed, name='kl_loss_1_fixed') variables = list(tf.trainable_variables()) self._loss = self._surrogate_loss self._grads_and_vars, self._policy_gradient = self.get_vars_grads( [self._surrogate_loss], variables) offset = 0 list_tangents = [] list_assigns = [] for variable in variables: shape = get_shape(variable) size = np.prod(shape) list_tangents.append(tf.reshape(tangents[offset:offset + size], shape)) list_assigns.append(tf.assign(variable, tf.reshape(theta[offset:offset + size], shape))) offset += size gradients = tf.gradients(self._kl_loss_1_fixed, variables) gradient_vector_product = [tf.reduce_sum(g * t) for (g, t) in zip(gradients, list_tangents)] _, self._fisher_vector_product = self.get_vars_grads(gradient_vector_product, variables) self._set_theta = tf.group(*list_assigns) self._get_theta = tf.concat(axis=0, values=[tf.reshape(variable, (-1,)) for variable in variables]) return self._losses, self._loss
def _build(self, incoming, *args, **kwargs): """ Args: incoming: `Tensor`. 3-D Tensor [samples, timesteps, input dim]. """ self._declare_dependencies() sequence_length = None if self.dynamic: sequence_length = retrieve_seq_length_op(incoming if isinstance( incoming, tf.Tensor) else tf.stack(incoming)) input_shape = get_shape(incoming) inference = incoming # If a tensor given, convert it to a per timestep list if type(inference) not in [list, np.array]: ndim = len(input_shape) assert ndim >= 3, 'Input dim should be at least 3.' axes = [1, 0] + list(range(2, ndim)) inference = tf.transpose(inference, (axes)) inference = tf.unstack(value=inference) if self.dynamic: outputs, state = tf.nn.dynamic_rnn( cell=self._cell, inputs=inference, dtype=tf.float32, initial_state=self.initial_state, sequence_length=sequence_length, scope=self.module_name) else: outputs, state = rnn.static_rnn(cell=self._cell, inputs=inference, dtype=tf.float32, initial_state=self.initial_state, sequence_length=sequence_length, scope=self.module_name) for v in [self._cell.w, self._cell.b]: if hasattr(v, '__len__'): for var in v: track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name) else: track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name) track(outputs[-1], tf.GraphKeys.ACTIVATIONS, self.module_name) if self.dynamic: if self.return_seq: o = outputs else: outputs = tf.transpose(tf.stack(outputs), [1, 0, 2]) o = advanced_indexing_op(outputs, sequence_length) else: o = outputs if self.return_seq else outputs[-1] track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name) return (o, state) if self.return_state else o
def _build(self, incoming, *args, **kwargs): """ Args: incoming: `Tensor`. 3-D Tensor Layer [samples, timesteps, input dim]. """ assert (self.rnncell_fw.output_size == self.rnncell_bw.output_size), "RNN Cells number of units must match!" input_shape = get_shape(incoming) # TODO: DropoutWrapper inference = incoming # If a tensor given, convert it to a per timestep list if type(inference) not in [list, np.array]: ndim = len(input_shape) assert ndim >= 3, 'Input dim should be at least 3.' axes = [1, 0] + list(xrange(2, ndim)) inference = tf.transpose(inference, (axes,)) inference = tf.unstack(inference) sequence_length = None if self.dynamic: sequence_length = retrieve_seq_length_op( incoming if isinstance(incoming, tf.Tensor) else tf.stack(incoming)) outputs, states_fw, states_bw = tf.nn.bidirectional_dynamic_rnn( cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference, initial_state_fw=self.initial_state_fw, initial_state_bw=self.initial_state_bw, sequence_length=sequence_length, dtype=tf.float32) else: outputs, states_fw, states_bw = rnn.static_bidirectional_rnn( cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference, initial_state_fw=self.initial_state_fw, initial_state_bw=self.initial_state_bw, dtype=tf.float32) for v in [self.rnncell_fw.w, self.rnncell_fw.b, self.rnncell_bw.w, self.rnncell_bw.b]: if hasattr(v, '__len__'): for var in v: track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name) else: track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1]) if self.dynamic: if self.return_seq: o = outputs else: outputs = tf.transpose(tf.stack(outputs), [1, 0, 2]) o = advanced_indexing_op(outputs, sequence_length) else: o = outputs if self.return_seq else outputs[-1] track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name) return (o, states_fw, states_bw) if self.return_states else o
def _build(self, incoming, *args, **kwargs): if Modes.is_train(self.mode): incoming = validate_dtype(incoming) input_shape = get_shape(incoming) x = incoming + self.scale * tf.random_normal( input_shape[1:], mean=self.mean, stddev=self.stddev, dtype=incoming.dtype, seed=self.seed) return x return incoming
def advanced_indexing_op(input, index): """Advanced Indexing for Sequences. """ batch_size = get_shape(input)[0] max_length = int(input.get_shape()[1]) dim_size = int(input.get_shape()[2]) index = tf.range(0, batch_size) * max_length + (index - 1) flat = tf.reshape(input, [-1, dim_size]) relevant = tf.gather(flat, index) return relevant
def _build(self, incoming, *args, **kwargs): """ Args: incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten. Returns: 2D Tensor [samples, num_units]. """ self._declare_dependencies() input_shape = get_shape(incoming) incoming = validate_dtype(incoming) assert len( input_shape) > 1, 'Incoming Tensor shape must be at least 2-D' n_inputs = total_tensor_depth(tensor_shape=input_shape) regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True) self._w = variable(name='w', shape=[n_inputs, self.num_units], dtype=incoming.dtype, regularizer=regularizer, initializer=getters.get_initializer( self.weights_init), trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 2: inference = tf.reshape(tensor=inference, shape=[-1, n_inputs]) inference = tf.matmul(a=inference, b=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[self.num_units], dtype=incoming.dtype, initializer=getters.get_initializer( self.bias_init), trainable=self.trainable, restore=self.restore) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.nn.bias_add(value=inference, bias=self._b) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) if self._dropout: inference = self._dropout(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): """ Args: incoming: `Tensor`. 3-D Tensor [samples, timesteps, input dim]. """ self._declare_dependencies() sequence_length = kwargs.get('sequence_length') if self.dynamic and sequence_length is None: sequence_length = retrieve_seq_length_op( incoming if isinstance(incoming, tf.Tensor) else tf.stack(incoming)) input_shape = get_shape(incoming) inference = incoming # If a static rnn and tensor given, convert it to a per timestep list if type(inference) not in [list, np.array] and not self.dynamic: ndim = len(input_shape) assert ndim >= 3, 'Input dim should be at least 3.' axes = [1, 0] + list(xrange(2, ndim)) inference = tf.transpose(inference, axes) inference = tf.unstack(value=inference) if self.dynamic: outputs, state = tf.nn.dynamic_rnn( cell=self._cell, inputs=inference, dtype=tf.float32, initial_state=self.initial_state, sequence_length=sequence_length, scope=self.module_name) else: outputs, state = rnn.static_rnn( cell=self._cell, inputs=inference, dtype=tf.float32, initial_state=self.initial_state, sequence_length=sequence_length, scope=self.module_name) for v in [self._cell.w, self._cell.b]: if hasattr(v, '__len__'): for var in v: track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name) else: track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name) track(outputs[-1], tf.GraphKeys.ACTIVATIONS, self.module_name) if self.dynamic: if self.return_seq: o = outputs else: o = get_sequence_relevant_output(outputs, sequence_length) else: o = outputs if self.return_seq else outputs[-1] track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name) return (o, state) if self.return_state else o
def _prelu(x, name): with get_name_scope(name): if channel_shared: w_shape = (1,) else: w_shape = get_shape(x)[-1:] w_init = getters.get_initializer(weights_init) alphas = variable(shape=w_shape, initializer=w_init, restore=restore, name="alphas") x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5 x.alphas = alphas return x
def _prelu(x, name): with get_name_scope(name): if channel_shared: w_shape = (1,) else: w_shape = get_shape(x)[-1:] W_init = getters.get_initializer(weights_init) alphas = variable(shape=w_shape, initializer=W_init, restore=restore, name="alphas") x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5 x.alphas = alphas return x
def _build(self, incoming, *args, **kwargs): """ Args: incoming: (2+)-D `Tensor`. Returns: 2-D `Tensor` [batch, flatten_dims]. """ input_shape = get_shape(incoming) assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D' dims = total_tensor_depth(tensor_shape=input_shape) x = tf.reshape(tensor=incoming, shape=[-1, dims]) track(x, tf.GraphKeys.LAYER_TENSOR, self.name) return x
def _build(self, incoming, *args, **kwargs): """ Args: incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten. Returns: 2D Tensor [samples, num_units]. """ self._declare_dependencies() input_shape = get_shape(incoming) assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D' n_inputs = total_tensor_depth(tensor_shape=input_shape) regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True) initializer = getters.get_initializer(self.weights_init) self._w = variable(name='w', shape=[n_inputs, self.num_units], regularizer=regularizer, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) self._b = variable(name='b', shape=[self.num_units], initializer=getters.get_initializer(self.bias_init), trainable=self.trainable, restore=self.restore) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) # Weight and bias for the transform gate self._w_t = variable(name='w_t', shape=[n_inputs, self.num_units], regularizer=None, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name) self._b_t = variable(name='b_t', shape=[self.num_units], initializer=tf.constant_initializer(-1), trainable=self.trainable, restore=self.restore) track(self._b_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name) # If input is not 2d, flatten it. if len(input_shape) > 2: incoming = tf.reshape(tensor=incoming, shape=[-1, n_inputs]) H = getters.get_activation(self.activation)(tf.matmul(a=incoming, b=self._w) + self._b) T = tf.sigmoid(tf.matmul(a=incoming, b=self._w_t) + self._b_t) if self._transform_dropout: T = self._transform_dropout(T) C = tf.subtract(x=1.0, y=T) inference = tf.add(x=tf.multiply(x=H, y=T), y=tf.multiply(x=incoming, y=C)) track(inference, tf.GraphKeys.ACTIVATIONS) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): input_shape = get_shape(incoming) input_ndim = len(input_shape) gamma_init = tf.random_normal_initializer(mean=self.gamma, stddev=self.stddev) self._beta = variable(name='beta', shape=[input_shape[-1]], initializer=tf.constant_initializer(self.beta), trainable=self.trainable, restore=self.restore) self._gamma = variable(name='gamma', shape=[input_shape[-1]], initializer=gamma_init, trainable=self.trainable, restore=self.restore) track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name) track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if not self.restore: track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta) track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma) axis = list(xrange(input_ndim - 1)) moving_mean = variable(name='moving_mean', shape=input_shape[-1:], initializer=tf.zeros_initializer(), trainable=False, restore=self.restore) moving_variance = variable(name='moving_variance', shape=input_shape[-1:], initializer=tf.constant_initializer(1.), trainable=False, restore=self.restore) def update_mean_var(): mean, variance = tf.nn.moments(x=incoming, axes=axis) update_moving_mean = moving_averages.assign_moving_average( variable=moving_mean, value=mean, decay=self.decay, zero_debias=False) update_moving_variance = moving_averages.assign_moving_average( variable=moving_variance, value=variance, decay=self.decay, zero_debias=False) with tf.control_dependencies([update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) # Retrieve variable managing training mode if Modes.is_train(self.mode): mean, var = update_mean_var() else: mean, var = moving_mean, moving_variance incoming = tf.nn.batch_normalization(x=incoming, mean=mean, variance=var, offset=self._beta, scale=self._gamma, variance_epsilon=self.epsilon) incoming.set_shape(input_shape) track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name) return incoming
def encode(self, features, labels, encoder_fn, *args, **kwargs): """Encodes the incoming tensor. Args: features: `Tensor`. labels: `dict` or `Tensor` encoder_fn: `function`. *args: **kwargs: """ if 'labels' in get_arguments(encoder_fn): kwargs['labels'] = labels output = encoder_fn(mode=self.mode, features=features, **kwargs) if self.state_size is None: self.state_size = get_shape(output)[1:] return output
def filp(images, axis=0, is_random=False, seed=None): """Flip (randomly) an image/images. (A mirror to tf.image flip_left_right, flip_up_down, random_flip_left_right, and random_flip_up_down) if axis is 0: * flip horizontally (left to right) if axis is 1: * vertically (upside down). Outputs the contents of `images` flipped along the given axis. Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. axis: `int`. 0 for horizontal, 1 for vertical is_random: `bool`, If True, flip randomly. seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}. Returns: If `image` was 4-D, a 4-D float Tensor of shape `[batch, target_height, target_width, channels]` If `image` was 3-D, a 3-D float Tensor of shape `[target_height, target_width, channels] Raises: ValueError: if the shape of `image` not supported. """ if axis == 0: method = tf.image.flip_left_right if not is_random else tf.image.random_flip_left_right elif axis == 1: method = tf.image.flip_up_down if not is_random else tf.image.random_flip_up_down else: raise ValueError("`axis` should be 0 or 1, received ``".format(axis)) images_shape = get_shape(images) if images_shape > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if images_shape == 4: if is_random: return tf.map_fn(lambda img: method(img, seed), images) return tf.map_fn(lambda img: method(img), images) return method(images)
def flip(images, axis=0, is_random=False, seed=None): """Flip (randomly) an image/images. (A mirror to tf.image flip_left_right, flip_up_down, random_flip_left_right, and random_flip_up_down) if axis is 0: * flip horizontally (left to right) if axis is 1: * vertically (upside down). Outputs the contents of `images` flipped along the given axis. Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. axis: `int`. 0 for horizontal, 1 for vertical is_random: `bool`, If True, flip randomly. seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}. Returns: If `image` was 4-D, a 4-D float Tensor of shape `[batch, target_height, target_width, channels]` If `image` was 3-D, a 3-D float Tensor of shape `[target_height, target_width, channels] Raises: ValueError: if the shape of `image` not supported. """ if axis == 0: method = tf.image.flip_left_right if not is_random else tf.image.random_flip_left_right elif axis == 1: method = tf.image.flip_up_down if not is_random else tf.image.random_flip_up_down else: raise ValueError("`axis` should be 0 or 1, received ``".format(axis)) images_shape = get_shape(images) if len(images_shape) > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if len(images_shape) == 4: if is_random: return tf.map_fn(lambda img: method(img, seed), images) return tf.map_fn(lambda img: method(img), images) return method(images)
def _build(self, incoming, *args, **kwargs): """ Args: incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten. Returns: 2D Tensor [samples, num_units]. """ self._declare_dependencies() input_shape = get_shape(incoming) incoming = validate_dtype(incoming) assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D' n_inputs = total_tensor_depth(tensor_shape=input_shape) regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True) self._w = variable( name='w', shape=[n_inputs, self.num_units], dtype=incoming.dtype, regularizer=regularizer, initializer=getters.get_initializer(self.weights_init), trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 2: inference = tf.reshape(tensor=inference, shape=[-1, n_inputs]) inference = tf.matmul(a=inference, b=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[self.num_units], dtype=incoming.dtype, initializer=getters.get_initializer(self.bias_init), trainable=self.trainable, restore=self.restore) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.nn.bias_add(value=inference, bias=self._b) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) if self._dropout: inference = self._dropout(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): """ Args: incoming: 1-D Tensor [samples]. If not 2D, input will be flatten. Returns: 1-D Tensor [samples]. """ input_shape = get_shape(incoming) n_inputs = int(np.prod(a=input_shape[1:])) initializer = tf.constant_initializer(value=np.random.randn()) self._w = variable(name='w', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 1: inference = tf.reshape(tensor=inference, shape=[-1]) inference = tf.multiply(x=inference, y=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) inference = tf.add(inference, self._b) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build_eval_metrics(self, results, features, labels): """Creates the loss operation Returns a tuple `(losses, loss)`: `losses` are the per-batch losses. `loss` is a single scalar tensor to minimize. """ lshape = get_shape(labels) if self.model_type == self.Types.CLASSIFIER: if len(lshape) == 1 or (len(lshape) and int(lshape[1]) == 1): results = tf.argmax(results) labels = tf.argmax(labels) else: results = tf.argmax(results, 1) labels = tf.argmax(labels, 1) metrics = {} for metric in self.eval_metrics_config: metrics[metric.name] = getters.get_eval_metric( metric.name, results, labels, **metric.params) return metrics
def central_crop(images, central_fraction): """Crop the central region of the image. (A mirror to tf.image central_crop) Remove the outer parts of an image but retain the central region of the image along each dimension. If we specify central_fraction = 0.5, this function returns the region marked with "X" in the below diagram. ``` -------- |........| |..XXXX..| |..XXXX..| |........| where "X" is the central 50% of the image. -------- ``` Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. central_fraction: float (0, 1], fraction of size to crop Raises: ValueError: if central_crop_fraction is not within (0, 1]. Returns: If `images` was 4-D, a 4-D float Tensor of shape `[batch, new_height, new_width, channels]`. If `images` was 3-D, a 3-D float Tensor of shape `[new_height, new_width, channels]`. """ images_shape = get_shape(images) if len(images_shape) > 4: ValueError("'image' must have either 3 or 4 dimensions, " "received `{}`.".format(images_shape)) if len(images_shape) == 4: return tf.map_fn(lambda img: tf.image.central_crop(img, central_fraction), images) return tf.image.central_crop(images, central_fraction)
def convert_images_dtype(images, dtype, saturate=False, name=None): """Convert image(s) to `dtype`, scaling its values if needed. (A mirror to tf.image convert_image_dtype) Images that are represented using floating point values are expected to have values in the range [0,1). Image data stored in integer data types are expected to have values in the range `[0,MAX]`, where `MAX` is the largest positive representable number for the data type. This op converts between data types, scaling the values appropriately before casting. Note that converting from floating point inputs to integer types may lead to over/underflow problems. Set saturate to `True` to avoid such problem in problematic conversions. If enabled, saturation will clip the output into the allowed range before performing a potentially dangerous cast (and only before performing such a cast, i.e., when casting from a floating point to an integer type, and when casting from a signed to an unsigned type; `saturate` has no effect on casts between floats, or on casts that increase the type's range). Args: images: An image. dtype: A `DType` to convert `image` to. saturate: If `True`, clip the input before casting (if necessary). name: A name for this operation (optional). Returns: `image`, converted to `dtype`. """ images_shape = get_shape(images) if len(images_shape) > 4: ValueError("'image' must have either 3 or 4 dimensions, " "received `{}`.".format(images_shape)) if len(images_shape) == 4: return tf.map_fn(lambda img: tf.image.convert_image_dtype( img, dtype=dtype, saturate=saturate, name=name), images) return tf.image.convert_image_dtype(images, dtype=dtype, saturate=saturate, name=name)
def convert_image_dtype(images, dtype, saturate=False, name=None): """Convert image(s) to `dtype`, scaling its values if needed. (A mirror to tf.image convert_image_dtype) Images that are represented using floating point values are expected to have values in the range [0,1). Image data stored in integer data types are expected to have values in the range `[0,MAX]`, where `MAX` is the largest positive representable number for the data type. This op converts between data types, scaling the values appropriately before casting. Note that converting from floating point inputs to integer types may lead to over/underflow problems. Set saturate to `True` to avoid such problem in problematic conversions. If enabled, saturation will clip the output into the allowed range before performing a potentially dangerous cast (and only before performing such a cast, i.e., when casting from a floating point to an integer type, and when casting from a signed to an unsigned type; `saturate` has no effect on casts between floats, or on casts that increase the type's range). Args: images: An image. dtype: A `DType` to convert `image` to. saturate: If `True`, clip the input before casting (if necessary). name: A name for this operation (optional). Returns: `image`, converted to `dtype`. """ images_shape = get_shape(images) if images_shape > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if images_shape == 4: return tf.map_fn(lambda img: tf.image.convert_image_dtype( img, dtype=dtype, saturate=saturate, name=name), images) return tf.image.convert_image_dtype(images, dtype=dtype, saturate=saturate, name=name)
def _build_eval_metrics(self, results, features, labels): """Creates the loss operation Returns a tuple `(losses, loss)`: `losses` are the per-batch losses. `loss` is a single scalar tensor to minimize. """ lshape = get_shape(labels) def get_labels_and_results(results, labels): if len(lshape) == 1 or (len(lshape) and int(lshape[1]) == 1): return tf.argmax(results), tf.argmax(labels) else: return tf.argmax(results, 1), tf.argmax(labels, 1) metrics = {} for metric in self.eval_metrics_config: _results, _labels = results, labels if self.model_type == self.Types.CLASSIFIER and metric.module in ARGMAX_METRICS: _results, _labels = get_labels_and_results(results, labels) metrics[metric.module] = getters.get_eval_metric( metric.module, _results, _labels, **metric.params) return metrics
def random_crop(images, height, width): """Randomly crops an image/images to a given size. Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. height: `float`. The height to crop to. width: `float`. The width to crop to. Returns: If `images` was 4-D, a 4-D float Tensor of shape `[batch, new_height, new_width, channels]`. If `images` was 3-D, a 3-D float Tensor of shape `[new_height, new_width, channels]`. """ images_shape = get_shape(images) if len(images_shape) > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if len(images_shape) == 4: return tf.map_fn(lambda img: tf.random_crop(img, [height, width, images_shape[-1]]), images) return tf.random_crop(images, [height, width, images_shape[-1]])
def _build_eval_metrics(self, results, features, labels): """Creates the loss operation Returns a tuple `(losses, loss)`: `losses` are the per-batch losses. `loss` is a single scalar tensor to minimize. """ lshape = get_shape(labels) def get_labels_and_results(results, labels): if len(lshape) == 1 or (len(lshape) and int(lshape[1]) == 1): return tf.argmax(results), tf.argmax(labels) else: return tf.argmax(results, 1), tf.argmax(labels, 1) metrics = {} for metric in self.eval_metrics_config: _results, _labels = results, labels if self.model_type == self.Types.CLASSIFIER and metric.module in ARGMAX_METRICS: _results, _labels = get_labels_and_results(results, labels) metrics[metric.module] = getters.get_eval_metric( metric.module, _results, _labels, **metric.params) return metrics
def _build(self, incoming, *args, **kwargs): """ Args: 2-D Tensor [samples, ids]. Returns: 3-D Tensor [samples, embedded_ids, features]. """ input_shape = get_shape(incoming) assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D' weights_init = getters.get_initializer(self.weights_init) self._w = variable('w', shape=[self.input_dim, self.output_dim], initializer=weights_init, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.cast(x=incoming, dtype=tf.int32) inference = tf.nn.embedding_lookup(params=self._w, ids=inference, validate_indices=self.validate_indices) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def central_crop(images, central_fraction): """Crop the central region of the image. (A mirror to tf.image central_crop) Remove the outer parts of an image but retain the central region of the image along each dimension. If we specify central_fraction = 0.5, this function returns the region marked with "X" in the below diagram. -------- | | | XXXX | | XXXX | | | where "X" is the central 50% of the image. -------- Args: images: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. central_fraction: float (0, 1], fraction of size to crop Raises: ValueError: if central_crop_fraction is not within (0, 1]. Returns: If `images` was 4-D, a 4-D float Tensor of shape `[batch, new_height, new_width, channels]`. If `images` was 3-D, a 3-D float Tensor of shape `[new_height, new_width, channels]`. """ images_shape = get_shape(images) if images_shape > 4: ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape)) if images_shape == 4: return tf.map_fn(lambda img: tf.image.central_crop(img, central_fraction), images) return tf.image.central_crop(images, central_fraction)
def _build(self, incoming, *args, **kwargs): """ Args: incoming: 1-D Tensor [samples]. If not 2D, input will be flatten. Returns: 1-D Tensor [samples]. """ input_shape = get_shape(incoming) n_inputs = total_tensor_depth(tensor_shape=input_shape) initializer = tf.constant_initializer(value=np.random.randn()) self._w = variable(name='w', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 1: inference = tf.reshape(tensor=inference, shape=[-1]) inference = tf.multiply(x=inference, y=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) inference = tf.add(inference, self._b) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def output_size(self, incoming): return get_shape(incoming)[1:]
def get_labels_and_results(results, labels): lshape = get_shape(labels) if len(lshape) == 1 or (lshape and int(lshape[1]) == 1): return tf.argmax(results), tf.argmax(labels) else: return tf.argmax(results, 1), tf.argmax(labels, 1)
def output_size(self, incoming): return get_shape(incoming)[1:]
def _build(self, incoming, *args, **kwargs): """ Args: incoming: `Tensor`. 3-D Tensor Layer [samples, timesteps, input dim]. """ assert (self.rnncell_fw.output_size == self.rnncell_bw.output_size ), "RNN Cells number of units must match!" sequence_length = kwargs.get('sequence_length') if self.dynamic and sequence_length is None: sequence_length = retrieve_seq_length_op(incoming if isinstance( incoming, tf.Tensor) else tf.stack(incoming)) input_shape = get_shape(incoming) # TODO: DropoutWrapper inference = incoming # If a static rnn and tensor given, convert it to a per timestep list if type(inference) not in [list, np.array] and not self.dynamic: ndim = len(input_shape) assert ndim >= 3, 'Input dim should be at least 3.' axes = [1, 0] + list(xrange(2, ndim)) inference = tf.transpose(inference, axes) inference = tf.unstack(value=inference) if self.dynamic: # outputs are a tuple of (fw, bw) outputs outputs, (states_fw, states_bw) = tf.nn.bidirectional_dynamic_rnn( cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference, initial_state_fw=self.initial_state_fw, initial_state_bw=self.initial_state_bw, sequence_length=sequence_length, dtype=tf.float32) else: # outputs are a concatenation of both bw and fw outputs outputs, states_fw, states_bw = rnn.static_bidirectional_rnn( cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference, initial_state_fw=self.initial_state_fw, initial_state_bw=self.initial_state_bw, sequence_length=sequence_length, dtype=tf.float32) for v in [ self.rnncell_fw.w, self.rnncell_fw.b, self.rnncell_bw.w, self.rnncell_bw.b ]: if hasattr(v, '__len__'): for var in v: track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name) else: track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if self.dynamic: tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[0][-1]) else: tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1]) if self.dynamic: if self.return_seq: o = outputs else: # we are only interested in the fw pass here o = get_sequence_relevant_output(outputs[0], sequence_length) else: o = outputs if self.return_seq else outputs[-1] track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name) return (o, states_fw, states_bw) if self.return_states else o
def _build(self, incoming, *args, **kwargs): input_shape = get_shape(incoming) input_ndim = len(input_shape) gamma_init = tf.random_normal_initializer(mean=self.gamma, stddev=self.stddev) self._beta = variable(name='beta', shape=[input_shape[-1]], initializer=tf.constant_initializer(self.beta), trainable=self.trainable, restore=self.restore) self._gamma = variable(name='gamma', shape=[input_shape[-1]], initializer=gamma_init, trainable=self.trainable, restore=self.restore) track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name) track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if not self.restore: track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta) track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma) axis = list(xrange(input_ndim - 1)) moving_mean = variable(name='moving_mean', shape=input_shape[-1:], initializer=tf.zeros_initializer(), trainable=False, restore=self.restore) moving_variance = variable(name='moving_variance', shape=input_shape[-1:], initializer=tf.constant_initializer(1.), trainable=False, restore=self.restore) def update_mean_var(): mean, variance = tf.nn.moments(x=incoming, axes=axis) update_moving_mean = moving_averages.assign_moving_average( variable=moving_mean, value=mean, decay=self.decay, zero_debias=False) update_moving_variance = moving_averages.assign_moving_average( variable=moving_variance, value=variance, decay=self.decay, zero_debias=False) with tf.control_dependencies( [update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) # Retrieve variable managing training mode if Modes.is_train(self.mode): mean, var = update_mean_var() else: mean, var = moving_mean, moving_variance incoming = tf.nn.batch_normalization(x=incoming, mean=mean, variance=var, offset=self._beta, scale=self._gamma, variance_epsilon=self.epsilon) incoming.set_shape(input_shape) track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name) return incoming
def _build(self, incoming, *args, **kwargs): """ Args: incoming: `Tensor`. 3-D Tensor Layer [samples, timesteps, input dim]. """ assert (self.rnncell_fw.output_size == self.rnncell_bw.output_size), "RNN Cells number of units must match!" sequence_length = kwargs.get('sequence_length') if self.dynamic and sequence_length is None: sequence_length = retrieve_seq_length_op( incoming if isinstance(incoming, tf.Tensor) else tf.stack(incoming)) input_shape = get_shape(incoming) # TODO: DropoutWrapper inference = incoming # If a static rnn and tensor given, convert it to a per timestep list if type(inference) not in [list, np.array] and not self.dynamic: ndim = len(input_shape) assert ndim >= 3, 'Input dim should be at least 3.' axes = [1, 0] + list(xrange(2, ndim)) inference = tf.transpose(inference, axes) inference = tf.unstack(value=inference) if self.dynamic: # outputs are a tuple of (fw, bw) outputs outputs, (states_fw, states_bw) = tf.nn.bidirectional_dynamic_rnn( cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference, initial_state_fw=self.initial_state_fw, initial_state_bw=self.initial_state_bw, sequence_length=sequence_length, dtype=tf.float32) else: # outputs are a concatenation of both bw and fw outputs outputs, states_fw, states_bw = rnn.static_bidirectional_rnn( cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference, initial_state_fw=self.initial_state_fw, initial_state_bw=self.initial_state_bw, sequence_length=sequence_length, dtype=tf.float32) for v in [self.rnncell_fw.w, self.rnncell_fw.b, self.rnncell_bw.w, self.rnncell_bw.b]: if hasattr(v, '__len__'): for var in v: track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name) else: track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if self.dynamic: tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[0][-1]) else: tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1]) if self.dynamic: if self.return_seq: o = outputs else: # we are only interested in the fw pass here o = get_sequence_relevant_output(outputs[0], sequence_length) else: o = outputs if self.return_seq else outputs[-1] track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name) return (o, states_fw, states_bw) if self.return_states else o