def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
incoming = validate_dtype(incoming)
assert len(
input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer,
scale=self.scale,
collect=True)
self._w = variable(name='w',
shape=[n_inputs, self.num_units],
dtype=incoming.dtype,
regularizer=regularizer,
initializer=getters.get_initializer(
self.weights_init),
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(tensor=inference, shape=[-1, n_inputs])
inference = tf.matmul(a=inference, b=self._w)
self._b = None
if self.bias:
self._b = variable(name='b',
shape=[self.num_units],
dtype=incoming.dtype,
initializer=getters.get_initializer(
self.bias_init),
trainable=self.trainable,
restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.nn.bias_add(value=inference, bias=self._b)
if self.activation:
inference = getters.get_activation(self.activation,
collect=True)(inference)
if self._dropout:
inference = self._dropout(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True)
initializer = getters.get_initializer(self.weights_init)
self._w = variable(name='w', shape=[n_inputs, self.num_units], regularizer=regularizer,
initializer=initializer, trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
self._b = variable(name='b', shape=[self.num_units],
initializer=getters.get_initializer(self.bias_init),
trainable=self.trainable, restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
# Weight and bias for the transform gate
self._w_t = variable(name='w_t', shape=[n_inputs, self.num_units],
regularizer=None, initializer=initializer,
trainable=self.trainable, restore=self.restore)
track(self._w_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
self._b_t = variable(name='b_t', shape=[self.num_units],
initializer=tf.constant_initializer(-1),
trainable=self.trainable, restore=self.restore)
track(self._b_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
# If input is not 2d, flatten it.
if len(input_shape) > 2:
incoming = tf.reshape(tensor=incoming, shape=[-1, n_inputs])
H = getters.get_activation(self.activation)(tf.matmul(a=incoming, b=self._w) + self._b)
T = tf.sigmoid(tf.matmul(a=incoming, b=self._w_t) + self._b_t)
if self._transform_dropout:
T = self._transform_dropout(T)
C = tf.subtract(x=1.0, y=T)
inference = tf.add(x=tf.multiply(x=H, y=T), y=tf.multiply(x=incoming, y=C))
track(inference, tf.GraphKeys.ACTIVATIONS)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _linear(args, output_size, bias, bias_start=0.0, weights_init=None,
trainable=True, restore=True, scope=None):
"""Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
Args:
args: a 2D Tensor or a list of 2D, batch x n, Tensors.
output_size: int, second dimension of W[i].
bias: boolean, whether to add a bias term or not.
bias_start: starting value to initialize the bias; 0 by default.
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A 2D Tensor with shape [batch x output_size] equal to
sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
Raises:
ValueError: if some of the arguments has unspecified or wrong shape.
"""
if args is None or (nest.is_sequence(args) and not args):
raise ValueError('`args` must be specified')
if not nest.is_sequence(args):
args = [args]
# Calculate the total size of arguments on dimension 1.
total_arg_size = 0
shapes = [a.get_shape().as_list() for a in args]
for shape in shapes:
if len(shape) != 2:
raise ValueError('Linear is expecting 2D arguments: %s' % str(shapes))
if not shape[1]:
raise ValueError('Linear expects shape[1] of arguments: %s' % str(shapes))
else:
total_arg_size += shape[1]
# Now the computation.
with get_variable_scope(scope or 'Linear'):
_w = variable(name='w', shape=[total_arg_size, output_size], initializer=weights_init,
trainable=trainable, restore=restore)
if len(args) == 1:
res = tf.matmul(a=args[0], b=_w)
else:
res = tf.matmul(a=array_ops.concat(values=args, axis=1), b=_w)
if not bias:
return res
_b = variable(name='b', shape=[output_size],
initializer=tf.constant_initializer(bias_start),
trainable=trainable, restore=restore)
return res + _b
def _linear(args, output_size, bias, bias_start=0.0, weights_init=None,
trainable=True, restore=True, scope=None):
"""Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
Args:
args: a 2D Tensor or a list of 2D, batch x n, Tensors.
output_size: int, second dimension of W[i].
bias: boolean, whether to add a bias term or not.
bias_start: starting value to initialize the bias; 0 by default.
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A 2D Tensor with shape [batch x output_size] equal to
sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
Raises:
ValueError: if some of the arguments has unspecified or wrong shape.
"""
if args is None or (nest.is_sequence(args) and not args):
raise ValueError('`args` must be specified')
if not nest.is_sequence(args):
args = [args]
# Calculate the total size of arguments on dimension 1.
total_arg_size = 0
shapes = [a.get_shape().as_list() for a in args]
for shape in shapes:
if len(shape) != 2:
raise ValueError('Linear is expecting 2D arguments: %s' % str(shapes))
if not shape[1]:
raise ValueError('Linear expects shape[1] of arguments: %s' % str(shapes))
else:
total_arg_size += shape[1]
# Now the computation.
with get_variable_scope(scope or 'Linear'):
_w = variable(name='w', shape=[total_arg_size, output_size], initializer=weights_init,
trainable=trainable, restore=restore)
if len(args) == 1:
res = tf.matmul(a=args[0], b=_w)
else:
res = tf.matmul(a=array_ops.concat(values=args, axis=1), b=_w)
if not bias:
return res
_b = variable(name='b', shape=[output_size],
initializer=tf.constant_initializer(bias_start),
trainable=trainable, restore=restore)
return res + _b
def _build(self, incoming, *args, **kwargs):
input_shape = get_shape(incoming)
input_ndim = len(input_shape)
gamma_init = tf.random_normal_initializer(mean=self.gamma, stddev=self.stddev)
self._beta = variable(name='beta', shape=[input_shape[-1]],
initializer=tf.constant_initializer(self.beta),
trainable=self.trainable, restore=self.restore)
self._gamma = variable(name='gamma', shape=[input_shape[-1]],
initializer=gamma_init, trainable=self.trainable,
restore=self.restore)
track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if not self.restore:
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta)
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma)
axis = list(xrange(input_ndim - 1))
moving_mean = variable(name='moving_mean', shape=input_shape[-1:],
initializer=tf.zeros_initializer(), trainable=False,
restore=self.restore)
moving_variance = variable(name='moving_variance', shape=input_shape[-1:],
initializer=tf.constant_initializer(1.), trainable=False,
restore=self.restore)
def update_mean_var():
mean, variance = tf.nn.moments(x=incoming, axes=axis)
update_moving_mean = moving_averages.assign_moving_average(
variable=moving_mean, value=mean, decay=self.decay, zero_debias=False)
update_moving_variance = moving_averages.assign_moving_average(
variable=moving_variance, value=variance, decay=self.decay, zero_debias=False)
with tf.control_dependencies([update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
# Retrieve variable managing training mode
if Modes.is_train(self.mode):
mean, var = update_mean_var()
else:
mean, var = moving_mean, moving_variance
incoming = tf.nn.batch_normalization(x=incoming, mean=mean, variance=var, offset=self._beta,
scale=self._gamma, variance_epsilon=self.epsilon)
incoming.set_shape(input_shape)
track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return incoming
def _build(self, incoming, *args, **kwargs):
"""
Args:
2-D Tensor [samples, ids].
Returns:
3-D Tensor [samples, embedded_ids, features].
"""
input_shape = get_shape(incoming)
assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D'
weights_init = getters.get_initializer(self.weights_init)
self._w = variable('w',
shape=[self.input_dim, self.output_dim],
initializer=weights_init,
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.cast(x=incoming, dtype=tf.int32)
inference = tf.nn.embedding_lookup(
params=self._w,
ids=inference,
validate_indices=self.validate_indices)
# Embedding doesn't support masking, so we save sequence length prior to the lookup.
# Expand dim to 3d.
shape = [-1] + inference.get_shape().as_list()[1:3] + [1]
inference.seq_length = retrieve_seq_length_op(
tf.reshape(incoming, shape))
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
2-D Tensor [samples, ids].
Returns:
3-D Tensor [samples, embedded_ids, features].
"""
input_shape = get_shape(incoming)
assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D'
weights_init = getters.get_initializer(self.weights_init)
self._w = variable('w',
shape=[self.input_dim, self.output_dim],
initializer=weights_init,
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.cast(x=incoming, dtype=tf.int32)
inference = tf.nn.embedding_lookup(
params=self._w,
ids=inference,
validate_indices=self.validate_indices)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def create_global_counter(collection, name, graph=None):
"""Create global counter tensor in graph.
Args:
collection: the counter's collection.
name: the counter's name.
graph: The graph in which to create the global counter tensor. If missing,
use default graph.
Returns:
Global step tensor.
Raises:
ValueError: if global counter tensor is already defined.
"""
graph = graph or tf.get_default_graph()
if get_global_counter(collection, name, graph) is not None:
raise ValueError("`{}` already exists.".format(collection))
# Create in proper graph and base name_scope.
with graph.as_default() as g, g.name_scope(None):
return variable(
collection,
shape=[],
dtype=tf.int64,
initializer=getters.get_initializer('zeros', dtype=tf.int64),
trainable=False,
collections=[tf.GraphKeys.GLOBAL_VARIABLES, collection])
def create_global_counter(collection, name, graph=None):
"""Create global counter tensor in graph.
Args:
collection: the counter's collection.
name: the counter's name.
graph: The graph in which to create the global counter tensor. If missing,
use default graph.
Returns:
Global step tensor.
Raises:
ValueError: if global counter tensor is already defined.
"""
graph = graph or tf.get_default_graph()
if get_global_counter(collection, name, graph) is not None:
raise ValueError("`{}` already exists.".format(collection))
# Create in proper graph and base name_scope.
with graph.as_default() as g, g.name_scope(None):
return variable(
collection,
shape=[],
dtype=tf.int64,
initializer=getters.get_initializer('zeros', dtype=tf.int64),
trainable=False,
collections=[tf.GraphKeys.GLOBAL_VARIABLES, collection])
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
incoming = validate_dtype(incoming)
assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True)
self._w = variable(
name='w', shape=[n_inputs, self.num_units], dtype=incoming.dtype, regularizer=regularizer,
initializer=getters.get_initializer(self.weights_init), trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(tensor=inference, shape=[-1, n_inputs])
inference = tf.matmul(a=inference, b=self._w)
self._b = None
if self.bias:
self._b = variable(name='b', shape=[self.num_units], dtype=incoming.dtype,
initializer=getters.get_initializer(self.bias_init),
trainable=self.trainable, restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.nn.bias_add(value=inference, bias=self._b)
if self.activation:
inference = getters.get_activation(self.activation, collect=True)(inference)
if self._dropout:
inference = self._dropout(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: 1-D Tensor [samples]. If not 2D, input will be flatten.
Returns:
1-D Tensor [samples].
"""
input_shape = get_shape(incoming)
n_inputs = int(np.prod(a=input_shape[1:]))
initializer = tf.constant_initializer(value=np.random.randn())
self._w = variable(name='w',
shape=[n_inputs],
dtype=incoming.dtype,
initializer=initializer,
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(tensor=inference, shape=[-1])
inference = tf.multiply(x=inference, y=self._w)
self._b = None
if self.bias:
self._b = variable(name='b',
shape=[n_inputs],
dtype=incoming.dtype,
initializer=initializer,
trainable=self.trainable,
restore=self.restore)
inference = tf.add(inference, self._b)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.activation:
inference = getters.get_activation(self.activation,
collect=True)(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _prelu(x, name):
with get_name_scope(name):
if channel_shared:
w_shape = (1,)
else:
w_shape = get_shape(x)[-1:]
w_init = getters.get_initializer(weights_init)
alphas = variable(shape=w_shape, initializer=w_init, restore=restore, name="alphas")
x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5
x.alphas = alphas
return x
def _prelu(x, name):
with get_name_scope(name):
if channel_shared:
w_shape = (1,)
else:
w_shape = get_shape(x)[-1:]
W_init = getters.get_initializer(weights_init)
alphas = variable(shape=w_shape, initializer=W_init, restore=restore, name="alphas")
x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5
x.alphas = alphas
return x
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: 1-D Tensor [samples]. If not 2D, input will be flatten.
Returns:
1-D Tensor [samples].
"""
input_shape = get_shape(incoming)
n_inputs = total_tensor_depth(tensor_shape=input_shape)
initializer = tf.constant_initializer(value=np.random.randn())
self._w = variable(name='w', shape=[n_inputs],
dtype=incoming.dtype, initializer=initializer,
trainable=self.trainable, restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(tensor=inference, shape=[-1])
inference = tf.multiply(x=inference, y=self._w)
self._b = None
if self.bias:
self._b = variable(name='b', shape=[n_inputs],
dtype=incoming.dtype, initializer=initializer,
trainable=self.trainable, restore=self.restore)
inference = tf.add(inference, self._b)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.activation:
inference = getters.get_activation(self.activation, collect=True)(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
2-D Tensor [samples, ids].
Returns:
3-D Tensor [samples, embedded_ids, features].
"""
input_shape = get_shape(incoming)
assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D'
weights_init = getters.get_initializer(self.weights_init)
self._w = variable('w', shape=[self.input_dim, self.output_dim],
initializer=weights_init,
trainable=self.trainable, restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.cast(x=incoming, dtype=tf.int32)
inference = tf.nn.embedding_lookup(params=self._w, ids=inference,
validate_indices=self.validate_indices)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
input_shape = get_shape(incoming)
input_ndim = len(input_shape)
gamma_init = tf.random_normal_initializer(mean=self.gamma,
stddev=self.stddev)
self._beta = variable(name='beta',
shape=[input_shape[-1]],
initializer=tf.constant_initializer(self.beta),
trainable=self.trainable,
restore=self.restore)
self._gamma = variable(name='gamma',
shape=[input_shape[-1]],
initializer=gamma_init,
trainable=self.trainable,
restore=self.restore)
track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if not self.restore:
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta)
track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma)
axis = list(xrange(input_ndim - 1))
moving_mean = variable(name='moving_mean',
shape=input_shape[-1:],
initializer=tf.zeros_initializer(),
trainable=False,
restore=self.restore)
moving_variance = variable(name='moving_variance',
shape=input_shape[-1:],
initializer=tf.constant_initializer(1.),
trainable=False,
restore=self.restore)
def update_mean_var():
mean, variance = tf.nn.moments(x=incoming, axes=axis)
update_moving_mean = moving_averages.assign_moving_average(
variable=moving_mean,
value=mean,
decay=self.decay,
zero_debias=False)
update_moving_variance = moving_averages.assign_moving_average(
variable=moving_variance,
value=variance,
decay=self.decay,
zero_debias=False)
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
# Retrieve variable managing training mode
if Modes.is_train(self.mode):
mean, var = update_mean_var()
else:
mean, var = moving_mean, moving_variance
incoming = tf.nn.batch_normalization(x=incoming,
mean=mean,
variance=var,
offset=self._beta,
scale=self._gamma,
variance_epsilon=self.epsilon)
incoming.set_shape(input_shape)
track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return incoming
