def _fill_and_one_pad_stride(stride, n):
"""Expands the provided stride to size n and pads it with 1s."""
try:
return (1, ) + _fill_shape(stride, n) + (1, )
except TypeError:
raise base.IncompatibleShapeError(
"stride is {} ({}), must be either an integer or an iterable of "
"integers of size {}".format(stride, type(stride), n))
def _build(self, inputs):
"""Connects the Linear module into the graph, with input Tensor `inputs`.
If this is not the first time the module has been connected to the graph,
the Tensor provided here must have the same final dimension, in order for
the existing variables to be the correct size for the multiplication. The
batch size may differ for each connection.
Args:
inputs: A 2D Tensor of size [batch_size, input_size].
Returns:
A 2D Tensor of size [batch_size, output_size].
Raises:
base.IncompatibleShapeError: If the input is not a 2-D `Tensor` with
the size of the second dimension specified.
base.IncompatibleShapeError: If reconnecting an already connected module
into the graph, and the shape of the input is not compatible with
previous inputs.
"""
input_shape = tuple(inputs.get_shape().as_list())
if len(input_shape) != 2:
raise base.IncompatibleShapeError(
"{}: rank of shape must be 2 not: {}".format(
self.name, len(input_shape)))
if input_shape[1] is None:
raise base.IncompatibleShapeError(
"{}: Input size must be specified at module build time".format(
self.name))
if self._input_shape is not None and input_shape[
1] != self._input_shape[1]:
raise base.IncompatibleShapeError(
"{}: Input shape must be [batch_size, {}] not: [batch_size, {}]"
.format(self.name, self._input_shape[1], input_shape[1]))
self._input_shape = input_shape
if "w" not in self._initializers:
self._initializers["w"] = create_linear_initializer(
self._input_shape[1])
if "b" not in self._initializers and self._use_bias:
self._initializers["b"] = create_linear_initializer(
self._input_shape[1])
weight_shape = (self._input_shape[1], self.output_size)
dtype = inputs.dtype
self._w = tf.get_variable("w",
shape=weight_shape,
dtype=dtype,
initializer=self._initializers["w"],
partitioner=self._partitioners.get(
"w", None))
outputs = tf.matmul(inputs, self._w)
if self._use_bias:
bias_shape = (self.output_size, )
self._b = tf.get_variable("b",
shape=bias_shape,
dtype=dtype,
initializer=self._initializers["b"],
partitioner=self._partitioners.get(
"b", None))
outputs += self._b
return outputs
def _build(self, input_batch, is_training=True, test_local_stats=True):
"""Connects the BatchNorm module into the graph.
Args:
input_batch: A Tensor of arbitrary dimension. By default, the final
dimension is not reduced over when computing the minibatch statistics.
is_training: A boolean to indicate if the module should be connected in
training mode, meaning the moving averages are updated. By default
`True`. Can be a Tensor.
test_local_stats: A boolean to indicate if local batch statistics should
be used when `is_training=False`. If not, moving averages are used.
By default `True`. Can be a Tensor.
Returns:
A tensor with the same shape as `input_batch`.
Raises:
base.IncompatibleShapeError: If `reduction_indices` is not valid for the
input shape or has negative entries.
base.NotSupportedError: If `input_batch` has data type of `tf.float16`.
"""
input_shape = input_batch.get_shape()
if self._reduction_indices is not None:
if len(self._reduction_indices) > len(input_shape):
raise base.IncompatibleShapeError(
"Too many reduction indices specified.")
if max(self._reduction_indices) >= len(input_shape):
raise base.IncompatibleShapeError(
"Reduction index too large for input shape.")
if min(self._reduction_indices) < 0:
raise base.IncompatibleShapeError(
"Reduction indeces must be non-negative.")
reduction_indices = self._reduction_indices
else:
# Reduce over all dimensions except the last.
reduction_indices = range(len(input_shape))[:-1]
if input_batch.dtype == tf.float16:
raise base.NotSupportedError(
"BatchNorm does not support `tf.float16`, insufficient "
"precision for calculating sufficient statistics.")
self._mean_shape = input_batch.get_shape().as_list()
for index in reduction_indices:
self._mean_shape[index] = 1
use_batch_stats = is_training | test_local_stats
# Use the legacy moving second moment if the flag is set.
if self._use_legacy_moving_second_moment:
tf.logging.warning(
"nn.BatchNorm `use_legacy_second_moment=True` is deprecated.")
mean, variance, second_moment = self._build_statistics_second_moment(
input_batch,
reduction_indices,
use_batch_stats)
self._build_update_ops_second_moment(mean, second_moment, is_training)
else:
mean, variance = self._build_statistics_variance(
input_batch,
reduction_indices,
use_batch_stats)
self._build_update_ops_variance(mean, variance, is_training)
# Set up optional scale and offset factors.
if self._offset:
self._set_default_initializer(self.BETA)
self._beta = tf.get_variable(
self.BETA,
shape=self._mean_shape,
initializer=self._initializers[self.BETA])
else:
self._beta = None
if self._scale:
self._set_default_initializer(self.GAMMA)
self._gamma = tf.get_variable(
self.GAMMA,
shape=self._mean_shape,
initializer=self._initializers[self.GAMMA])
else:
self._gamma = None
out = tf.nn.batch_normalization(
input_batch,
mean,
variance,
self._beta,
self._gamma,
self._eps,
name="batch_norm")
return out
def _fill_and_verify_kernel_shape(x, n):
"""Expands x if necessary into a `n`-D kernel shape and reports errors."""
try:
return _fill_shape(x, n)
except TypeError as e:
raise base.IncompatibleShapeError("Invalid kernel shape: {}".format(e))
def _build(self, inputs):
"""Connects the Conv2DTranspose module into the graph.
If this is not the first time the module has been connected to the graph,
the input Tensor provided here must have the same final 3 dimensions, in
order for the existing variables to be the correct size for the
multiplication. The batch size may differ for each connection.
Args:
inputs: A 4D Tensor of shape [batch_size, input_height, input_width,
input_channels].
Returns:
A 4D Tensor of shape [batch_size, output_height, output_width,
output_channels].
Raises:
ValueError: If connecting the module into the graph any time after the
first time and the inferred size of the input does not match previous
invocations.
base.IncompatibleShapeError: If the input tensor has the wrong number of
dimensions; or if the input tensor has an unknown `input_channels`; or
or if `output_shape` is an iterable and is not in the format
`(out_height, out_width)`.
TypeError: If input Tensor dtype is not `tf.float32`.
"""
# Handle input whose shape is unknown during graph creation.
self._input_shape = tuple(inputs.get_shape().as_list())
if len(self._input_shape) != 4:
raise base.IncompatibleShapeError(
"Input Tensor must have shape (batch_size, input_height, "
"input_width, input_channels)")
if self._input_shape[3] is None:
raise base.IncompatibleShapeError(
"Number of input channels must be known at module build time")
input_channels = self._input_shape[3]
if inputs.dtype != tf.float32:
raise TypeError(
"Input must have dtype tf.float32, but dtype was " +
inputs.dtype)
if len(self.output_shape) != 2:
raise base.IncompatibleShapeError(
"Output shape must be specified as "
"(output_height, output_width)")
weight_shape = (self._kernel_shape[0], self._kernel_shape[1],
self.output_channels, input_channels)
bias_shape = (self.output_channels, )
if "w" not in self._initializers:
fan_in_shape = weight_shape[:2] + (weight_shape[3], )
self._initializers["w"] = create_weight_initializer(fan_in_shape)
if "b" not in self._initializers and self._use_bias:
self._initializers["b"] = create_bias_initializer(bias_shape)
self._w = tf.get_variable("w",
shape=weight_shape,
initializer=self._initializers["w"])
# Use tensorflow shape op to manipulate inputs shape, so that unknown batch
# size - which can happen when using input placeholders - is handled
# correcly.
batch_size = tf.expand_dims(tf.shape(inputs)[0], 0)
conv_output_shape = tf.convert_to_tensor(
tuple(self.output_shape) + (self.output_channels, ))
output_shape = tf.concat(0, [batch_size, conv_output_shape])
outputs = tf.nn.conv2d_transpose(inputs,
self._w,
output_shape,
strides=self._stride,
padding=self._padding)
if self._use_bias:
self._b = tf.get_variable("b",
shape=bias_shape,
initializer=self._initializers["b"])
outputs += self._b
# Recover output tensor shape value and pass it to set_shape in order to
# enable shape inference.
batch_size_value = inputs.get_shape()[0]
output_shape_value = ((batch_size_value, ) + self.output_shape +
(self.output_channels, ))
outputs.set_shape(output_shape_value)
return outputs
def __init__(self,
output_channels,
output_shape,
kernel_shape,
stride=1,
padding=SAME,
use_bias=True,
initializers=None,
name="conv_2d_transpose"):
"""Constructs a `Conv2DTranspose module`.
See the following documentation for an explanation of VALID versus SAME
padding modes:
https://www.tensorflow.org/versions/r0.8/api_docs/python/nn.html#convolution
Args:
output_channels: Number of output channels.
Can be either a number or a callable. In the latter case, since the
function invocation is deferred to graph construction time, the user
must only ensure `output_channels` can be called, returning an
integer, when build is called.
output_shape: Output shape of transpose convolution.
Can be either an iterable of integers or a callable. In the latter
case, since the function invocation is deferred to graph construction
time, the user must only ensure that `output_shape` can be called,
returning an iterable of format `(out_height, out_width)` when
`_build` is called. Note that `output_shape` defines the size of
output signal domain, as opposed to the shape of the output `Tensor`.
kernel_shape: List of kernel sizes, must be length 2.
stride: List of kernel strides.
padding: Padding algorithm, either `nn.SAME` or `nn.VALID`.
use_bias: Whether to include bias parameters. Default `True`.
initializers: Optional dict containing ops to initialize the filters (with
key 'w') or biases (with key 'b').
name: Name of the module.
Raises:
base.IncompatibleShapeError: If the given kernel shape is neither an
integer nor a sequence of two integers.
base.IncompatibleShapeError: If the given stride is neither an integer nor
a sequence of two or four integers.
ValueError: If the given padding is not `nn.VALID` or `nn.SAME`.
KeyError: If `initializers` contains any keys other than 'w' or 'b'.
TypeError: If any of the given initializers are not callable.
"""
super(Conv2DTranspose, self).__init__(name)
self._output_channels = output_channels
if callable(output_shape):
self._output_shape = output_shape
else:
self._output_shape = tuple(output_shape)
self._input_shape = None
self._kernel_shape = _fill_and_verify_kernel_shape(kernel_shape, 2)
# We want to support passing native strides akin to [1, m, n, 1].
if isinstance(stride, collections.Iterable) and len(stride) == 4:
if not stride[0] == stride[3] == 1:
raise base.IncompatibleShapeError(
"Invalid stride: First and last element must be 1.")
self._stride = tuple(stride)
else:
self._stride = _fill_and_one_pad_stride(stride, 2)
self._padding = _verify_padding(padding)
self._use_bias = use_bias
self.possible_keys = self.get_possible_initializer_keys(
use_bias=use_bias)
self._initializers = util.check_initializers(initializers,
self.possible_keys)
def _build(self, inputs):
"""Connects the Conv2D module into the graph, with input Tensor `inputs`.
If this is not the first time the module has been connected to the graph,
the input Tensor provided here must have the same final 3 dimensions, in
order for the existing variables to be the correct size for the
multiplication. The batch size may differ for each connection.
Args:
inputs: A 4D Tensor of shape [batch_size, input_height, input_width,
input_channels].
Returns:
A 4D Tensor of shape [batch_size, output_height, output_width,
output_channels].
Raises:
ValueError: If connecting the module into the graph any time after the
first time and the inferred size of the input does not match previous
invocations.
base.IncompatibleShapeError: If the input tensor has the wrong number
of dimensions.
base.IncompatibleShapeError: If a mask is present and its shape is
incompatible with the shape of the weights.
base.UnderspecifiedError: If the input tensor has an unknown
`input_channels`.
base.UnderspecifiedError: If rate > 1 is used with an input tensor with
unknown `input_width` or `input_height`.
TypeError: If input Tensor dtype is not `tf.float32`.
"""
# Handle input whose shape is unknown during graph creation.
self._input_shape = tuple(inputs.get_shape().as_list())
if len(self._input_shape) != 4:
raise base.IncompatibleShapeError(
"Input Tensor must have shape (batch_size, input_height, input_"
"width, input_channels)")
if self._input_shape[3] is None:
raise base.UnderSpecifiedError(
"Number of input channels must be known at module build time")
else:
input_channels = self._input_shape[3]
if inputs.dtype != tf.float32:
raise TypeError(
"Input must have dtype tf.float32, but dtype was {}".format(
inputs.dtype))
weight_shape = (self._kernel_shape[0], self._kernel_shape[1],
input_channels, self.output_channels)
bias_shape = (self.output_channels, )
if "w" not in self._initializers:
self._initializers["w"] = create_weight_initializer(
weight_shape[:3])
if "b" not in self._initializers and self._use_bias:
self._initializers["b"] = create_bias_initializer(bias_shape)
self._w = tf.get_variable("w",
shape=weight_shape,
initializer=self._initializers["w"])
w = self._w
if self._mask is not None:
mask_rank = self._mask.ndim
mask_shape = self._mask.shape
if mask_rank == 2:
if mask_shape != self._kernel_shape:
raise base.IncompatibleShapeError(
"Invalid mask shape: {}".format(mask_shape))
mask = np.reshape(self._mask, self._kernel_shape + (1, 1))
elif mask_rank == 4:
if mask_shape != tuple(weight_shape):
raise base.IncompatibleShapeError(
"Invalid mask shape: {}".format(mask_shape))
mask = self._mask
mask_tensor, = tf.py_func(lambda: mask, [], [w.dtype],
stateful=False)
mask_tensor.set_shape(weight_shape)
w *= mask
if self._rate > 1:
if any(x is None for x in self._input_shape[1:-1]):
raise base.UnderspecifiedError(
"Can't use atrous convolutions with unknown input_width or "
"input_height at graph build time")
outputs = tf.nn.atrous_conv2d(inputs,
w,
rate=self._rate,
padding=self._padding)
else:
outputs = tf.nn.conv2d(inputs,
w,
strides=self._stride,
padding=self._padding)
if self._use_bias:
self._b = tf.get_variable("b",
shape=bias_shape,
initializer=self._initializers["b"])
outputs += self._b
return outputs
def __init__(self,
output_channels,
kernel_shape,
stride=1,
rate=1,
padding=SAME,
use_bias=True,
initializers=None,
mask=None,
name="conv_2d"):
"""Constructs a Conv2D module.
See the following documentation for an explanation of VALID versus SAME
padding modes:
https://www.tensorflow.org/versions/r0.8/api_docs/python/nn.html#convolution
Args:
output_channels: Number of output channels. `output_channels` can be
either a number or a callable. In the latter case, since the function
invocation is deferred to graph construction time, the user must only
ensure that output_channels can be called, returning an integer,
when `_build` is called.
kernel_shape: List of kernel sizes, or integer that is used to define
kernel size in all dimensions.
stride: List of kernel strides, or integer that is used to define
stride in all dimensions.
rate: A positive integer, `rate=1` corresponds to standard 2D convolution,
`rate > 1` corresponds to dilated convolution.
padding: Padding algorithm, either `nn.SAME` or `nn.VALID`.
use_bias: Whether to include bias parameters. Default `True`.
initializers: Optional dict containing ops to initialize the filters (with
key 'w') or biases (with key 'b'). The default initializers are
truncated normal initializers, which are commonly used when the inputs
are zero centered (see https://arxiv.org/pdf/1502.03167v3.pdf).
mask: Optional 2D or 4D array, tuple or numpy array containing values to
multiply the weights by component-wise.
name: Name of the module.
Raises:
base.IncompatibleShapeError: If the given kernel shape is not an integer;
or if the given kernel shape is not a sequence of two integers.
base.IncompatibleShapeError: If the given stride is not an integer; or if
the given stride is not a sequence of two or four integers.
base.IncompatibleShapeError: If a mask is given and its rank is neither 2
nor 4.
base.NotSupportedError: If the given dilation rate is not a positive
integer.
base.NotSupportedError: If rate > 1 and the stride in any dimension is
> 1.
ValueError: If the given padding is not `nn.VALID` or `nn.SAME`.
KeyError: If initializers contains any keys other than 'w' or 'b'.
TypeError: If any of the given initializers are not callable.
TypeError: If mask is given and is not an array, tuple or a numpy array.
"""
super(Conv2D, self).__init__(name=name)
self._output_channels = output_channels
self._input_shape = None
self._kernel_shape = _fill_and_verify_kernel_shape(kernel_shape, 2)
try:
self._stride = (1, ) + _fill_shape(stride, 2) + (1, )
except TypeError as e:
# We want to support passing native strides akin to [1, m, n, 1].
if len(stride) == 4:
self._stride = tuple(stride)
else:
raise base.IncompatibleShapeError(
"Invalid stride: {}".format(e))
if not isinstance(rate, numbers.Integral) or rate < 1:
raise base.NotSupportedError(
"Rate, {}, must be integer >= 1".format(rate))
elif any(x > 1 for x in self._stride) and rate > 1:
raise base.NotSupportedError(
"Cannot have stride > 1 with rate > 1")
else:
self._rate = rate
self._padding = _verify_padding(padding)
self._use_bias = use_bias
self.possible_keys = self.get_possible_initializer_keys(
use_bias=use_bias)
self._initializers = util.check_initializers(initializers,
self.possible_keys)
if mask is not None:
if not isinstance(mask, (list, tuple, np.ndarray)):
raise TypeError("Invalid type for mask: {}".format(type(mask)))
self._mask = np.asanyarray(mask)
mask_rank = mask.ndim
if mask_rank != 2 and mask_rank != 4:
raise base.IncompatibleShapeError(
"Invalid mask rank: {}".format(mask_rank))
else:
self._mask = None