def get_sequence_relevant_output(input, sequence_length):
"""Returns the last relevant output for a sequence."""
input_shape = get_shape(input)
dynamic_input_shape = get_shape(input, dynamic=True)
batch_size = input_shape[0] or dynamic_input_shape[0]
max_length = input_shape[1] or dynamic_input_shape[1]
dim_size = input_shape[2] or dynamic_input_shape[2]
index = tf.range(batch_size) * max_length + (sequence_length - 1)
flat = tf.reshape(input, [-1, dim_size])
relevant = tf.gather(flat, index)
return relevant
def get_sequence_relevant_output(input, sequence_length):
"""Returns the last relevant output for a sequence."""
input_shape = get_shape(input)
dynamic_input_shape = get_shape(input, dynamic=True)
batch_size = input_shape[0] or dynamic_input_shape[0]
max_length = input_shape[1] or dynamic_input_shape[1]
dim_size = input_shape[2] or dynamic_input_shape[2]
index = tf.range(batch_size) * max_length + (sequence_length - 1)
flat = tf.reshape(input, [-1, dim_size])
relevant = tf.gather(flat, index)
return relevant
def standardize(images):
"""Linearly scales `image` to have zero mean and unit norm.
(A mirror to tf.image per_image_standardization)
This op computes `(x - mean) / adjusted_stddev`, where `mean` is the average
of all values in image, and
`adjusted_stddev = max(stddev, 1.0/sqrt(image.NumElements()))`.
`stddev` is the standard deviation of all values in `image`. It is capped
away from zero to protect against division by 0 when handling uniform images.
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
Returns:
The standardized image with same shape as `image`.
Raises:
ValueError: if the shape of 'image' is incompatible with this function.
"""
images_shape = get_shape(images)
if images_shape > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if images_shape == 4:
return tf.map_fn(lambda img: tf.image.per_image_standardization(img), images)
return tf.image.per_image_standardization(images)
def rotate90(images, k=1, is_random=False, seed=None, name=None):
"""Rotate (randomly) images counter-clockwise by 90 degrees.
(A mirror to tf.image rot90)
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
k: A scalar integer. The number of times the image is rotated by 90 degrees.
is_random: `bool`, If True, adjust randomly.
seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}.
name: A name for this operation (optional).
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
if is_random:
k = random_ops.random_shuffle([0, 1, 2, 3], seed=seed)[0]
images_shape = get_shape(images)
if images_shape > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if images_shape == 4:
return tf.map_fn(lambda img: tf.image.rot90(img, k, name), images)
return tf.image.rot90(images, k, name)
def decode(self, features, labels, decoder_fn, *args, **kwargs):
"""Decodes the incoming tensor if it's validates against the state size of the decoder.
Otherwise, generates a random value.
Args:
features: `Tensor`
labels: `dict` or `Tensor`
decoder_fn: `function`.
*args:
**kwargs:
"""
incoming_shape = get_shape(features)
if incoming_shape[1:] != self.state_size:
raise ValueError('`incoming` tensor is incompatible with decoder function, '
'expects a tensor with shape `{}`, '
'received instead `{}`'.format(self.state_size, incoming_shape[1:]))
# TODO: make decode capable of generating values directly,
# TODO: basically accepting None incoming values. Should also specify a distribution.
# shape = self._get_decoder_shape(incoming)
# return decoder_fn(mode=self.mode, inputs=tf.random_normal(shape=shape))
if 'labels' in get_arguments(decoder_fn):
kwargs['labels'] = labels
x = decoder_fn(mode=self.mode, features=features, **kwargs)
if not isinstance(x, DecoderSpec):
raise ValueError('`decoder_fn` should return an DecoderSpec.')
return x.output
def standardize(images):
"""Linearly scales `image` to have zero mean and unit norm.
(A mirror to tf.image per_image_standardization)
This op computes `(x - mean) / adjusted_stddev`, where `mean` is the average
of all values in image, and
`adjusted_stddev = max(stddev, 1.0/sqrt(image.NumElements()))`.
`stddev` is the standard deviation of all values in `image`. It is capped
away from zero to protect against division by 0 when handling uniform images.
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
Returns:
The standardized image with same shape as `image`.
Raises:
ValueError: if the shape of 'image' is incompatible with this function.
"""
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.per_image_standardization(img), images)
return tf.image.per_image_standardization(images)
def decode(self, incoming, decoder_fn, *args, **kwargs):
"""Decodes the incoming tensor if it's validates against the state size of the decoder.
Otherwise, generates a random value.
Args:
incoming: `Tensor`
decoder_fn: `function`.
*args:
**kwargs:
"""
incoming_shape = get_shape(incoming)
if incoming_shape[1:] != self.state_size:
raise ValueError(
'`incoming` tensor is incompatible with decoder function, '
'expects a tensor with shape `{}`, '
'received instead `{}`'.format(self.state_size,
incoming_shape[1:]))
# TODO: make decode capable of generating values directly,
# TODO: basically accecpting None incoming values. Should also specify a distribution.
# shape = self._get_decoder_shape(incoming)
# return decoder_fn(mode=self.mode, inputs=tf.random_normal(shape=shape))
x = decoder_fn(mode=self.mode, inputs=incoming)
if not isinstance(x, DecoderSpec):
raise ValueError('`decoder_fn` should return an DecoderSpec.')
return x.output
def transpose(images):
"""Transpose an image/images by swapping the first and second dimension.
(A mirror to tf.image transpose_image)
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.transpose_image(img), images)
return tf.image.transpose_image(images)
def _build(self, incoming, *args, **kwargs):
"""
Args:
2-D Tensor [samples, ids].
Returns:
3-D Tensor [samples, embedded_ids, features].
"""
input_shape = get_shape(incoming)
assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D'
weights_init = getters.get_initializer(self.weights_init)
self._w = variable('w',
shape=[self.input_dim, self.output_dim],
initializer=weights_init,
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.cast(x=incoming, dtype=tf.int32)
inference = tf.nn.embedding_lookup(
params=self._w,
ids=inference,
validate_indices=self.validate_indices)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
2-D Tensor [samples, ids].
Returns:
3-D Tensor [samples, embedded_ids, features].
"""
input_shape = get_shape(incoming)
assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D'
weights_init = getters.get_initializer(self.weights_init)
self._w = variable('w',
shape=[self.input_dim, self.output_dim],
initializer=weights_init,
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.cast(x=incoming, dtype=tf.int32)
inference = tf.nn.embedding_lookup(
params=self._w,
ids=inference,
validate_indices=self.validate_indices)
# Embedding doesn't support masking, so we save sequence length prior to the lookup.
# Expand dim to 3d.
shape = [-1] + inference.get_shape().as_list()[1:3] + [1]
inference.seq_length = retrieve_seq_length_op(
tf.reshape(incoming, shape))
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def rotate90(images, k=1, is_random=False, seed=None, name=None):
"""Rotate (randomly) images counter-clockwise by 90 degrees.
(A mirror to tf.image rot90)
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
k: A scalar integer. The number of times the image is rotated by 90 degrees.
is_random: `bool`, If True, adjust randomly.
seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}.
name: A name for this operation (optional).
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
if is_random:
k = random_ops.random_shuffle([0, 1, 2, 3], seed=seed)[0]
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.rot90(img, k, name), images)
return tf.image.rot90(images, k, name)
def transpose(images):
"""Transpose an image/images by swapping the first and second dimension.
(A mirror to tf.image transpose_image)
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
images_shape = get_shape(images)
if images_shape > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if images_shape == 4:
return tf.map_fn(lambda img: tf.image.transpose_image(img), images)
return tf.image.transpose_image(images)
def _build_loss(self, results, features, labels):
"""Creates the loss operation
Returns:
tuple `(losses, loss)`:
`losses` are the per-batch losses.
`loss` is a single scalar tensor to minimize.
"""
action = labels['action']
discount_reward = labels['discount_reward']
dist_values = labels['dist_values']
tangents = labels.get('tangents')
theta = labels.get('theta')
old_distribution = self._build_distribution(values=dist_values)
log_probs = self._graph_results.distribution.log_prob(action)
old_log_probs = old_distribution.log_prob(action)
self._losses = tf.multiply(x=tf.exp(log_probs - old_log_probs), y=discount_reward)
self._surrogate_loss = -tf.reduce_mean(self._losses, axis=0, name='surrogate_loss')
entropy = self._graph_results.distribution.entropy()
self._entropy_loss = tf.reduce_mean(entropy, name='entropy_loss')
kl_divergence_value = kl_divergence(self._graph_results.distribution, old_distribution)
self._kl_loss = tf.reduce_mean(kl_divergence_value, name='kl_loss')
if self.is_continuous:
dist_values_fixed = tf.stop_gradient(
tf.concat(values=[self._graph_results.distribution.loc,
self._graph_results.distribution.scale], axis=0))
else:
dist_values_fixed = tf.stop_gradient(self._graph_results.distribution.logits)
distribution_1_fixed = self._build_distribution(values=dist_values_fixed)
kl_divergence_1_fixed = kl_divergence(
distribution_1_fixed, self._graph_results.distribution)
self._kl_loss_1_fixed = tf.reduce_mean(kl_divergence_1_fixed, name='kl_loss_1_fixed')
variables = list(tf.trainable_variables())
self._loss = self._surrogate_loss
self._grads_and_vars, self._policy_gradient = self.get_vars_grads(
[self._surrogate_loss], variables)
offset = 0
list_tangents = []
list_assigns = []
for variable in variables:
shape = get_shape(variable)
size = np.prod(shape)
list_tangents.append(tf.reshape(tangents[offset:offset + size], shape))
list_assigns.append(tf.assign(variable, tf.reshape(theta[offset:offset + size], shape)))
offset += size
gradients = tf.gradients(self._kl_loss_1_fixed, variables)
gradient_vector_product = [tf.reduce_sum(g * t) for (g, t) in zip(gradients, list_tangents)]
_, self._fisher_vector_product = self.get_vars_grads(gradient_vector_product, variables)
self._set_theta = tf.group(*list_assigns)
self._get_theta = tf.concat(axis=0,
values=[tf.reshape(variable, (-1,)) for variable in variables])
return self._losses, self._loss
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: `Tensor`. 3-D Tensor [samples, timesteps, input dim].
"""
self._declare_dependencies()
sequence_length = None
if self.dynamic:
sequence_length = retrieve_seq_length_op(incoming if isinstance(
incoming, tf.Tensor) else tf.stack(incoming))
input_shape = get_shape(incoming)
inference = incoming
# If a tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array]:
ndim = len(input_shape)
assert ndim >= 3, 'Input dim should be at least 3.'
axes = [1, 0] + list(range(2, ndim))
inference = tf.transpose(inference, (axes))
inference = tf.unstack(value=inference)
if self.dynamic:
outputs, state = tf.nn.dynamic_rnn(
cell=self._cell,
inputs=inference,
dtype=tf.float32,
initial_state=self.initial_state,
sequence_length=sequence_length,
scope=self.module_name)
else:
outputs, state = rnn.static_rnn(cell=self._cell,
inputs=inference,
dtype=tf.float32,
initial_state=self.initial_state,
sequence_length=sequence_length,
scope=self.module_name)
for v in [self._cell.w, self._cell.b]:
if hasattr(v, '__len__'):
for var in v:
track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
else:
track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
track(outputs[-1], tf.GraphKeys.ACTIVATIONS, self.module_name)
if self.dynamic:
if self.return_seq:
o = outputs
else:
outputs = tf.transpose(tf.stack(outputs), [1, 0, 2])
o = advanced_indexing_op(outputs, sequence_length)
else:
o = outputs if self.return_seq else outputs[-1]
track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return (o, state) if self.return_state else o
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: `Tensor`. 3-D Tensor Layer [samples, timesteps, input dim].
"""
assert (self.rnncell_fw.output_size ==
self.rnncell_bw.output_size), "RNN Cells number of units must match!"
input_shape = get_shape(incoming)
# TODO: DropoutWrapper
inference = incoming
# If a tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array]:
ndim = len(input_shape)
assert ndim >= 3, 'Input dim should be at least 3.'
axes = [1, 0] + list(xrange(2, ndim))
inference = tf.transpose(inference, (axes,))
inference = tf.unstack(inference)
sequence_length = None
if self.dynamic:
sequence_length = retrieve_seq_length_op(
incoming if isinstance(incoming, tf.Tensor) else tf.stack(incoming))
outputs, states_fw, states_bw = tf.nn.bidirectional_dynamic_rnn(
cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference,
initial_state_fw=self.initial_state_fw,
initial_state_bw=self.initial_state_bw,
sequence_length=sequence_length,
dtype=tf.float32)
else:
outputs, states_fw, states_bw = rnn.static_bidirectional_rnn(
cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference,
initial_state_fw=self.initial_state_fw,
initial_state_bw=self.initial_state_bw,
dtype=tf.float32)
for v in [self.rnncell_fw.w, self.rnncell_fw.b, self.rnncell_bw.w, self.rnncell_bw.b]:
if hasattr(v, '__len__'):
for var in v:
track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
else:
track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1])
if self.dynamic:
if self.return_seq:
o = outputs
else:
outputs = tf.transpose(tf.stack(outputs), [1, 0, 2])
o = advanced_indexing_op(outputs, sequence_length)
else:
o = outputs if self.return_seq else outputs[-1]
track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return (o, states_fw, states_bw) if self.return_states else o
def _build(self, incoming, *args, **kwargs):
if Modes.is_train(self.mode):
incoming = validate_dtype(incoming)
input_shape = get_shape(incoming)
x = incoming + self.scale * tf.random_normal(
input_shape[1:],
mean=self.mean, stddev=self.stddev, dtype=incoming.dtype, seed=self.seed)
return x
return incoming
def advanced_indexing_op(input, index):
"""Advanced Indexing for Sequences. """
batch_size = get_shape(input)[0]
max_length = int(input.get_shape()[1])
dim_size = int(input.get_shape()[2])
index = tf.range(0, batch_size) * max_length + (index - 1)
flat = tf.reshape(input, [-1, dim_size])
relevant = tf.gather(flat, index)
return relevant
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
incoming = validate_dtype(incoming)
assert len(
input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer,
scale=self.scale,
collect=True)
self._w = variable(name='w',
shape=[n_inputs, self.num_units],
dtype=incoming.dtype,
regularizer=regularizer,
initializer=getters.get_initializer(
self.weights_init),
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(tensor=inference, shape=[-1, n_inputs])
inference = tf.matmul(a=inference, b=self._w)
self._b = None
if self.bias:
self._b = variable(name='b',
shape=[self.num_units],
dtype=incoming.dtype,
initializer=getters.get_initializer(
self.bias_init),
trainable=self.trainable,
restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.nn.bias_add(value=inference, bias=self._b)
if self.activation:
inference = getters.get_activation(self.activation,
collect=True)(inference)
if self._dropout:
inference = self._dropout(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: `Tensor`. 3-D Tensor [samples, timesteps, input dim].
"""
self._declare_dependencies()
sequence_length = kwargs.get('sequence_length')
if self.dynamic and sequence_length is None:
sequence_length = retrieve_seq_length_op(
incoming if isinstance(incoming, tf.Tensor) else tf.stack(incoming))
input_shape = get_shape(incoming)
inference = incoming
# If a static rnn and tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array] and not self.dynamic:
ndim = len(input_shape)
assert ndim >= 3, 'Input dim should be at least 3.'
axes = [1, 0] + list(xrange(2, ndim))
inference = tf.transpose(inference, axes)
inference = tf.unstack(value=inference)
if self.dynamic:
outputs, state = tf.nn.dynamic_rnn(
cell=self._cell, inputs=inference, dtype=tf.float32,
initial_state=self.initial_state, sequence_length=sequence_length,
scope=self.module_name)
else:
outputs, state = rnn.static_rnn(
cell=self._cell, inputs=inference, dtype=tf.float32,
initial_state=self.initial_state, sequence_length=sequence_length,
scope=self.module_name)
for v in [self._cell.w, self._cell.b]:
if hasattr(v, '__len__'):
for var in v:
track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
else:
track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
track(outputs[-1], tf.GraphKeys.ACTIVATIONS, self.module_name)
if self.dynamic:
if self.return_seq:
o = outputs
else:
o = get_sequence_relevant_output(outputs, sequence_length)
else:
o = outputs if self.return_seq else outputs[-1]
track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return (o, state) if self.return_state else o
def _prelu(x, name):
with get_name_scope(name):
if channel_shared:
w_shape = (1,)
else:
w_shape = get_shape(x)[-1:]
w_init = getters.get_initializer(weights_init)
alphas = variable(shape=w_shape, initializer=w_init, restore=restore, name="alphas")
x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5
x.alphas = alphas
return x
def _prelu(x, name):
with get_name_scope(name):
if channel_shared:
w_shape = (1,)
else:
w_shape = get_shape(x)[-1:]
W_init = getters.get_initializer(weights_init)
alphas = variable(shape=w_shape, initializer=W_init, restore=restore, name="alphas")
x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5
x.alphas = alphas
return x
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D `Tensor`.
Returns:
2-D `Tensor` [batch, flatten_dims].
"""
input_shape = get_shape(incoming)
assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
dims = total_tensor_depth(tensor_shape=input_shape)
x = tf.reshape(tensor=incoming, shape=[-1, dims])
track(x, tf.GraphKeys.LAYER_TENSOR, self.name)
return x
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True)
initializer = getters.get_initializer(self.weights_init)
self._w = variable(name='w', shape=[n_inputs, self.num_units], regularizer=regularizer,
initializer=initializer, trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
self._b = variable(name='b', shape=[self.num_units],
initializer=getters.get_initializer(self.bias_init),
trainable=self.trainable, restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
# Weight and bias for the transform gate
self._w_t = variable(name='w_t', shape=[n_inputs, self.num_units],
regularizer=None, initializer=initializer,
trainable=self.trainable, restore=self.restore)
track(self._w_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
self._b_t = variable(name='b_t', shape=[self.num_units],
initializer=tf.constant_initializer(-1),
trainable=self.trainable, restore=self.restore)
track(self._b_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
# If input is not 2d, flatten it.
if len(input_shape) > 2:
incoming = tf.reshape(tensor=incoming, shape=[-1, n_inputs])
H = getters.get_activation(self.activation)(tf.matmul(a=incoming, b=self._w) + self._b)
T = tf.sigmoid(tf.matmul(a=incoming, b=self._w_t) + self._b_t)
if self._transform_dropout:
T = self._transform_dropout(T)
C = tf.subtract(x=1.0, y=T)
inference = tf.add(x=tf.multiply(x=H, y=T), y=tf.multiply(x=incoming, y=C))
track(inference, tf.GraphKeys.ACTIVATIONS)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
input_shape = get_shape(incoming)
input_ndim = len(input_shape)
gamma_init = tf.random_normal_initializer(mean=self.gamma, stddev=self.stddev)
self._beta = variable(name='beta', shape=[input_shape[-1]],
initializer=tf.constant_initializer(self.beta),
trainable=self.trainable, restore=self.restore)
self._gamma = variable(name='gamma', shape=[input_shape[-1]],
initializer=gamma_init, trainable=self.trainable,
restore=self.restore)
track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if not self.restore:
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta)
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma)
axis = list(xrange(input_ndim - 1))
moving_mean = variable(name='moving_mean', shape=input_shape[-1:],
initializer=tf.zeros_initializer(), trainable=False,
restore=self.restore)
moving_variance = variable(name='moving_variance', shape=input_shape[-1:],
initializer=tf.constant_initializer(1.), trainable=False,
restore=self.restore)
def update_mean_var():
mean, variance = tf.nn.moments(x=incoming, axes=axis)
update_moving_mean = moving_averages.assign_moving_average(
variable=moving_mean, value=mean, decay=self.decay, zero_debias=False)
update_moving_variance = moving_averages.assign_moving_average(
variable=moving_variance, value=variance, decay=self.decay, zero_debias=False)
with tf.control_dependencies([update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
# Retrieve variable managing training mode
if Modes.is_train(self.mode):
mean, var = update_mean_var()
else:
mean, var = moving_mean, moving_variance
incoming = tf.nn.batch_normalization(x=incoming, mean=mean, variance=var, offset=self._beta,
scale=self._gamma, variance_epsilon=self.epsilon)
incoming.set_shape(input_shape)
track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return incoming
def encode(self, features, labels, encoder_fn, *args, **kwargs):
"""Encodes the incoming tensor.
Args:
features: `Tensor`.
labels: `dict` or `Tensor`
encoder_fn: `function`.
*args:
**kwargs:
"""
if 'labels' in get_arguments(encoder_fn):
kwargs['labels'] = labels
output = encoder_fn(mode=self.mode, features=features, **kwargs)
if self.state_size is None:
self.state_size = get_shape(output)[1:]
return output
def filp(images, axis=0, is_random=False, seed=None):
"""Flip (randomly) an image/images.
(A mirror to tf.image flip_left_right, flip_up_down, random_flip_left_right, and
random_flip_up_down)
if axis is 0:
* flip horizontally (left to right)
if axis is 1:
* vertically (upside down).
Outputs the contents of `images` flipped along the given axis.
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
axis: `int`. 0 for horizontal, 1 for vertical
is_random: `bool`, If True, flip randomly.
seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}.
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
if axis == 0:
method = tf.image.flip_left_right if not is_random else tf.image.random_flip_left_right
elif axis == 1:
method = tf.image.flip_up_down if not is_random else tf.image.random_flip_up_down
else:
raise ValueError("`axis` should be 0 or 1, received ``".format(axis))
images_shape = get_shape(images)
if images_shape > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if images_shape == 4:
if is_random:
return tf.map_fn(lambda img: method(img, seed), images)
return tf.map_fn(lambda img: method(img), images)
return method(images)
def flip(images, axis=0, is_random=False, seed=None):
"""Flip (randomly) an image/images.
(A mirror to tf.image flip_left_right, flip_up_down, random_flip_left_right, and
random_flip_up_down)
if axis is 0:
* flip horizontally (left to right)
if axis is 1:
* vertically (upside down).
Outputs the contents of `images` flipped along the given axis.
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
axis: `int`. 0 for horizontal, 1 for vertical
is_random: `bool`, If True, flip randomly.
seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}.
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
if axis == 0:
method = tf.image.flip_left_right if not is_random else tf.image.random_flip_left_right
elif axis == 1:
method = tf.image.flip_up_down if not is_random else tf.image.random_flip_up_down
else:
raise ValueError("`axis` should be 0 or 1, received ``".format(axis))
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if len(images_shape) == 4:
if is_random:
return tf.map_fn(lambda img: method(img, seed), images)
return tf.map_fn(lambda img: method(img), images)
return method(images)
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
incoming = validate_dtype(incoming)
assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True)
self._w = variable(
name='w', shape=[n_inputs, self.num_units], dtype=incoming.dtype, regularizer=regularizer,
initializer=getters.get_initializer(self.weights_init), trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(tensor=inference, shape=[-1, n_inputs])
inference = tf.matmul(a=inference, b=self._w)
self._b = None
if self.bias:
self._b = variable(name='b', shape=[self.num_units], dtype=incoming.dtype,
initializer=getters.get_initializer(self.bias_init),
trainable=self.trainable, restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.nn.bias_add(value=inference, bias=self._b)
if self.activation:
inference = getters.get_activation(self.activation, collect=True)(inference)
if self._dropout:
inference = self._dropout(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: 1-D Tensor [samples]. If not 2D, input will be flatten.
Returns:
1-D Tensor [samples].
"""
input_shape = get_shape(incoming)
n_inputs = int(np.prod(a=input_shape[1:]))
initializer = tf.constant_initializer(value=np.random.randn())
self._w = variable(name='w',
shape=[n_inputs],
dtype=incoming.dtype,
initializer=initializer,
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(tensor=inference, shape=[-1])
inference = tf.multiply(x=inference, y=self._w)
self._b = None
if self.bias:
self._b = variable(name='b',
shape=[n_inputs],
dtype=incoming.dtype,
initializer=initializer,
trainable=self.trainable,
restore=self.restore)
inference = tf.add(inference, self._b)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.activation:
inference = getters.get_activation(self.activation,
collect=True)(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build_eval_metrics(self, results, features, labels):
"""Creates the loss operation
Returns a tuple `(losses, loss)`:
`losses` are the per-batch losses.
`loss` is a single scalar tensor to minimize.
"""
lshape = get_shape(labels)
if self.model_type == self.Types.CLASSIFIER:
if len(lshape) == 1 or (len(lshape) and int(lshape[1]) == 1):
results = tf.argmax(results)
labels = tf.argmax(labels)
else:
results = tf.argmax(results, 1)
labels = tf.argmax(labels, 1)
metrics = {}
for metric in self.eval_metrics_config:
metrics[metric.name] = getters.get_eval_metric(
metric.name, results, labels, **metric.params)
return metrics
def central_crop(images, central_fraction):
"""Crop the central region of the image.
(A mirror to tf.image central_crop)
Remove the outer parts of an image but retain the central region of the image
along each dimension. If we specify central_fraction = 0.5, this function
returns the region marked with "X" in the below diagram.
```
--------
|........|
|..XXXX..|
|..XXXX..|
|........| where "X" is the central 50% of the image.
--------
```
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
central_fraction: float (0, 1], fraction of size to crop
Raises:
ValueError: if central_crop_fraction is not within (0, 1].
Returns:
If `images` was 4-D, a 4-D float Tensor of shape
`[batch, new_height, new_width, channels]`.
If `images` was 3-D, a 3-D float Tensor of shape
`[new_height, new_width, channels]`.
"""
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.central_crop(img, central_fraction), images)
return tf.image.central_crop(images, central_fraction)
def convert_images_dtype(images, dtype, saturate=False, name=None):
"""Convert image(s) to `dtype`, scaling its values if needed.
(A mirror to tf.image convert_image_dtype)
Images that are represented using floating point values are expected to have
values in the range [0,1). Image data stored in integer data types are
expected to have values in the range `[0,MAX]`, where `MAX` is the largest
positive representable number for the data type.
This op converts between data types, scaling the values appropriately before
casting.
Note that converting from floating point inputs to integer types may lead to
over/underflow problems. Set saturate to `True` to avoid such problem in
problematic conversions. If enabled, saturation will clip the output into the
allowed range before performing a potentially dangerous cast (and only before
performing such a cast, i.e., when casting from a floating point to an integer
type, and when casting from a signed to an unsigned type; `saturate` has no
effect on casts between floats, or on casts that increase the type's range).
Args:
images: An image.
dtype: A `DType` to convert `image` to.
saturate: If `True`, clip the input before casting (if necessary).
name: A name for this operation (optional).
Returns:
`image`, converted to `dtype`.
"""
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.convert_image_dtype(
img, dtype=dtype, saturate=saturate, name=name), images)
return tf.image.convert_image_dtype(images, dtype=dtype, saturate=saturate, name=name)
def convert_image_dtype(images, dtype, saturate=False, name=None):
"""Convert image(s) to `dtype`, scaling its values if needed.
(A mirror to tf.image convert_image_dtype)
Images that are represented using floating point values are expected to have
values in the range [0,1). Image data stored in integer data types are
expected to have values in the range `[0,MAX]`, where `MAX` is the largest
positive representable number for the data type.
This op converts between data types, scaling the values appropriately before
casting.
Note that converting from floating point inputs to integer types may lead to
over/underflow problems. Set saturate to `True` to avoid such problem in
problematic conversions. If enabled, saturation will clip the output into the
allowed range before performing a potentially dangerous cast (and only before
performing such a cast, i.e., when casting from a floating point to an integer
type, and when casting from a signed to an unsigned type; `saturate` has no
effect on casts between floats, or on casts that increase the type's range).
Args:
images: An image.
dtype: A `DType` to convert `image` to.
saturate: If `True`, clip the input before casting (if necessary).
name: A name for this operation (optional).
Returns:
`image`, converted to `dtype`.
"""
images_shape = get_shape(images)
if images_shape > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if images_shape == 4:
return tf.map_fn(lambda img: tf.image.convert_image_dtype(
img, dtype=dtype, saturate=saturate, name=name), images)
return tf.image.convert_image_dtype(images, dtype=dtype, saturate=saturate, name=name)
def _build_eval_metrics(self, results, features, labels):
"""Creates the loss operation
Returns a tuple `(losses, loss)`:
`losses` are the per-batch losses.
`loss` is a single scalar tensor to minimize.
"""
lshape = get_shape(labels)
def get_labels_and_results(results, labels):
if len(lshape) == 1 or (len(lshape) and int(lshape[1]) == 1):
return tf.argmax(results), tf.argmax(labels)
else:
return tf.argmax(results, 1), tf.argmax(labels, 1)
metrics = {}
for metric in self.eval_metrics_config:
_results, _labels = results, labels
if self.model_type == self.Types.CLASSIFIER and metric.module in ARGMAX_METRICS:
_results, _labels = get_labels_and_results(results, labels)
metrics[metric.module] = getters.get_eval_metric(
metric.module, _results, _labels, **metric.params)
return metrics
def random_crop(images, height, width):
"""Randomly crops an image/images to a given size.
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
height: `float`. The height to crop to.
width: `float`. The width to crop to.
Returns:
If `images` was 4-D, a 4-D float Tensor of shape
`[batch, new_height, new_width, channels]`.
If `images` was 3-D, a 3-D float Tensor of shape
`[new_height, new_width, channels]`.
"""
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.random_crop(img, [height, width, images_shape[-1]]), images)
return tf.random_crop(images, [height, width, images_shape[-1]])
def _build_eval_metrics(self, results, features, labels):
"""Creates the loss operation
Returns a tuple `(losses, loss)`:
`losses` are the per-batch losses.
`loss` is a single scalar tensor to minimize.
"""
lshape = get_shape(labels)
def get_labels_and_results(results, labels):
if len(lshape) == 1 or (len(lshape) and int(lshape[1]) == 1):
return tf.argmax(results), tf.argmax(labels)
else:
return tf.argmax(results, 1), tf.argmax(labels, 1)
metrics = {}
for metric in self.eval_metrics_config:
_results, _labels = results, labels
if self.model_type == self.Types.CLASSIFIER and metric.module in ARGMAX_METRICS:
_results, _labels = get_labels_and_results(results, labels)
metrics[metric.module] = getters.get_eval_metric(
metric.module, _results, _labels, **metric.params)
return metrics
def _build(self, incoming, *args, **kwargs):
"""
Args:
2-D Tensor [samples, ids].
Returns:
3-D Tensor [samples, embedded_ids, features].
"""
input_shape = get_shape(incoming)
assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D'
weights_init = getters.get_initializer(self.weights_init)
self._w = variable('w', shape=[self.input_dim, self.output_dim],
initializer=weights_init,
trainable=self.trainable, restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.cast(x=incoming, dtype=tf.int32)
inference = tf.nn.embedding_lookup(params=self._w, ids=inference,
validate_indices=self.validate_indices)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def central_crop(images, central_fraction):
"""Crop the central region of the image.
(A mirror to tf.image central_crop)
Remove the outer parts of an image but retain the central region of the image
along each dimension. If we specify central_fraction = 0.5, this function
returns the region marked with "X" in the below diagram.
--------
| |
| XXXX |
| XXXX |
| | where "X" is the central 50% of the image.
--------
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
central_fraction: float (0, 1], fraction of size to crop
Raises:
ValueError: if central_crop_fraction is not within (0, 1].
Returns:
If `images` was 4-D, a 4-D float Tensor of shape
`[batch, new_height, new_width, channels]`.
If `images` was 3-D, a 3-D float Tensor of shape
`[new_height, new_width, channels]`.
"""
images_shape = get_shape(images)
if images_shape > 4:
ValueError("'image' must have either 3 or 4 dimensions, received ``.".format(images_shape))
if images_shape == 4:
return tf.map_fn(lambda img: tf.image.central_crop(img, central_fraction), images)
return tf.image.central_crop(images, central_fraction)
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: 1-D Tensor [samples]. If not 2D, input will be flatten.
Returns:
1-D Tensor [samples].
"""
input_shape = get_shape(incoming)
n_inputs = total_tensor_depth(tensor_shape=input_shape)
initializer = tf.constant_initializer(value=np.random.randn())
self._w = variable(name='w', shape=[n_inputs],
dtype=incoming.dtype, initializer=initializer,
trainable=self.trainable, restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(tensor=inference, shape=[-1])
inference = tf.multiply(x=inference, y=self._w)
self._b = None
if self.bias:
self._b = variable(name='b', shape=[n_inputs],
dtype=incoming.dtype, initializer=initializer,
trainable=self.trainable, restore=self.restore)
inference = tf.add(inference, self._b)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.activation:
inference = getters.get_activation(self.activation, collect=True)(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def output_size(self, incoming):
return get_shape(incoming)[1:]
def get_labels_and_results(results, labels):
lshape = get_shape(labels)
if len(lshape) == 1 or (lshape and int(lshape[1]) == 1):
return tf.argmax(results), tf.argmax(labels)
else:
return tf.argmax(results, 1), tf.argmax(labels, 1)
def output_size(self, incoming):
return get_shape(incoming)[1:]
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: `Tensor`. 3-D Tensor Layer [samples, timesteps, input dim].
"""
assert (self.rnncell_fw.output_size == self.rnncell_bw.output_size
), "RNN Cells number of units must match!"
sequence_length = kwargs.get('sequence_length')
if self.dynamic and sequence_length is None:
sequence_length = retrieve_seq_length_op(incoming if isinstance(
incoming, tf.Tensor) else tf.stack(incoming))
input_shape = get_shape(incoming)
# TODO: DropoutWrapper
inference = incoming
# If a static rnn and tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array] and not self.dynamic:
ndim = len(input_shape)
assert ndim >= 3, 'Input dim should be at least 3.'
axes = [1, 0] + list(xrange(2, ndim))
inference = tf.transpose(inference, axes)
inference = tf.unstack(value=inference)
if self.dynamic:
# outputs are a tuple of (fw, bw) outputs
outputs, (states_fw, states_bw) = tf.nn.bidirectional_dynamic_rnn(
cell_fw=self.rnncell_fw,
cell_bw=self.rnncell_bw,
inputs=inference,
initial_state_fw=self.initial_state_fw,
initial_state_bw=self.initial_state_bw,
sequence_length=sequence_length,
dtype=tf.float32)
else:
# outputs are a concatenation of both bw and fw outputs
outputs, states_fw, states_bw = rnn.static_bidirectional_rnn(
cell_fw=self.rnncell_fw,
cell_bw=self.rnncell_bw,
inputs=inference,
initial_state_fw=self.initial_state_fw,
initial_state_bw=self.initial_state_bw,
sequence_length=sequence_length,
dtype=tf.float32)
for v in [
self.rnncell_fw.w, self.rnncell_fw.b, self.rnncell_bw.w,
self.rnncell_bw.b
]:
if hasattr(v, '__len__'):
for var in v:
track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
else:
track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.dynamic:
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[0][-1])
else:
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1])
if self.dynamic:
if self.return_seq:
o = outputs
else:
# we are only interested in the fw pass here
o = get_sequence_relevant_output(outputs[0], sequence_length)
else:
o = outputs if self.return_seq else outputs[-1]
track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return (o, states_fw, states_bw) if self.return_states else o
def _build(self, incoming, *args, **kwargs):
input_shape = get_shape(incoming)
input_ndim = len(input_shape)
gamma_init = tf.random_normal_initializer(mean=self.gamma,
stddev=self.stddev)
self._beta = variable(name='beta',
shape=[input_shape[-1]],
initializer=tf.constant_initializer(self.beta),
trainable=self.trainable,
restore=self.restore)
self._gamma = variable(name='gamma',
shape=[input_shape[-1]],
initializer=gamma_init,
trainable=self.trainable,
restore=self.restore)
track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if not self.restore:
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta)
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma)
axis = list(xrange(input_ndim - 1))
moving_mean = variable(name='moving_mean',
shape=input_shape[-1:],
initializer=tf.zeros_initializer(),
trainable=False,
restore=self.restore)
moving_variance = variable(name='moving_variance',
shape=input_shape[-1:],
initializer=tf.constant_initializer(1.),
trainable=False,
restore=self.restore)
def update_mean_var():
mean, variance = tf.nn.moments(x=incoming, axes=axis)
update_moving_mean = moving_averages.assign_moving_average(
variable=moving_mean,
value=mean,
decay=self.decay,
zero_debias=False)
update_moving_variance = moving_averages.assign_moving_average(
variable=moving_variance,
value=variance,
decay=self.decay,
zero_debias=False)
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
# Retrieve variable managing training mode
if Modes.is_train(self.mode):
mean, var = update_mean_var()
else:
mean, var = moving_mean, moving_variance
incoming = tf.nn.batch_normalization(x=incoming,
mean=mean,
variance=var,
offset=self._beta,
scale=self._gamma,
variance_epsilon=self.epsilon)
incoming.set_shape(input_shape)
track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return incoming
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: `Tensor`. 3-D Tensor Layer [samples, timesteps, input dim].
"""
assert (self.rnncell_fw.output_size ==
self.rnncell_bw.output_size), "RNN Cells number of units must match!"
sequence_length = kwargs.get('sequence_length')
if self.dynamic and sequence_length is None:
sequence_length = retrieve_seq_length_op(
incoming if isinstance(incoming, tf.Tensor) else tf.stack(incoming))
input_shape = get_shape(incoming)
# TODO: DropoutWrapper
inference = incoming
# If a static rnn and tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array] and not self.dynamic:
ndim = len(input_shape)
assert ndim >= 3, 'Input dim should be at least 3.'
axes = [1, 0] + list(xrange(2, ndim))
inference = tf.transpose(inference, axes)
inference = tf.unstack(value=inference)
if self.dynamic:
# outputs are a tuple of (fw, bw) outputs
outputs, (states_fw, states_bw) = tf.nn.bidirectional_dynamic_rnn(
cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference,
initial_state_fw=self.initial_state_fw,
initial_state_bw=self.initial_state_bw,
sequence_length=sequence_length,
dtype=tf.float32)
else:
# outputs are a concatenation of both bw and fw outputs
outputs, states_fw, states_bw = rnn.static_bidirectional_rnn(
cell_fw=self.rnncell_fw, cell_bw=self.rnncell_bw, inputs=inference,
initial_state_fw=self.initial_state_fw,
initial_state_bw=self.initial_state_bw,
sequence_length=sequence_length,
dtype=tf.float32)
for v in [self.rnncell_fw.w, self.rnncell_fw.b, self.rnncell_bw.w, self.rnncell_bw.b]:
if hasattr(v, '__len__'):
for var in v:
track(var, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
else:
track(v, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.dynamic:
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[0][-1])
else:
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1])
if self.dynamic:
if self.return_seq:
o = outputs
else:
# we are only interested in the fw pass here
o = get_sequence_relevant_output(outputs[0], sequence_length)
else:
o = outputs if self.return_seq else outputs[-1]
track(o, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return (o, states_fw, states_bw) if self.return_states else o