def _bias_variable(num_output_nodes,
dtype,
init=standard_ops.constant_initializer(0.),
collections=None,
create_summaries=True):
"""Create bias variable with shape `(num_output_nodes,)`.
Args:
num_output_nodes: The size of the output.
dtype: Data type of the variable.
init: An optional initialization. If not specified, use zeros.
collections: List of graph collections to which we'll add the new variable.
create_summaries: Set to false to disable summaries.
Returns:
The result weights variable.
"""
b = variable_scope.get_variable('bias',
shape=(num_output_nodes, ),
dtype=dtype,
initializer=init,
collections=collections)
if not variable_scope.get_variable_scope().reuse and create_summaries:
_add_histogram_summary(b)
return b
def _bias_variable(
num_output_nodes, dtype, init=standard_ops.constant_initializer(0.),
collections=None, create_summaries=True):
"""Create bias variable with shape `(num_output_nodes,)`.
Args:
num_output_nodes: The size of the output.
dtype: Data type of the variable.
init: An optional initialization. If not specified, use zeros.
collections: List of graph collections to which we'll add the new variable.
create_summaries: Set to false to disable summaries.
Returns:
The result weights variable.
"""
b = variable_scope.get_variable(
'bias', shape=(num_output_nodes,), dtype=dtype, initializer=init,
collections=collections)
if not variable_scope.get_variable_scope().reuse and create_summaries:
_add_histogram_summary(b)
return b
def fully_connected(x,
num_output_units,
activation_fn=None,
weight_init=initializers.xavier_initializer(),
bias_init=standard_ops.constant_initializer(0.),
name=None,
weight_collections=(ops.GraphKeys.WEIGHTS, ),
bias_collections=(ops.GraphKeys.BIASES, ),
output_collections=(ops.GraphKeys.ACTIVATIONS, ),
weight_regularizer=None,
bias_regularizer=None):
"""Adds the parameters for a fully connected layer and returns the output.
A fully connected layer is generally defined as a matrix multiply:
`y = f(w * x + b)` where `f` is given by `activation_fn`. If
`activation_fn` is `None`, the result of `y = w * x + b` is
returned.
This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting
`bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which in collections to place
the created variables (`weight_collections` and `bias_collections`; note that
the variables are always added to the `VARIABLES` collection). The output of
the layer can be placed in custom collections using `output_collections`.
The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`,
respectively.
A per layer regularization can be specified by setting `weight_regularizer`
and `bias_regularizer`, which are applied to the weights and biases
respectively, and whose output is added to the `REGULARIZATION_LOSSES`
collection.
Args:
x: The input `Tensor`.
num_output_units: The size of the output.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity. If None is used, do not apply any activation.
weight_init: An optional weight initialization, defaults to
`xavier_initializer`.
bias_init: An initializer for the bias, defaults to 0. Set to `None` in
order to disable bias.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "fully_connected" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections to which weights are added.
bias_collections: List of graph collections to which biases are added.
output_collections: List of graph collections to which outputs are added.
weight_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for weights.
bias_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for biases.
Returns:
The output of the fully connected layer.
"""
with variable_scope.variable_op_scope([x], name, 'fully_connected'):
num_input_units = x.get_shape().dims[1].value
dtype = x.dtype.base_dtype
w = _weight_variable(shape=[num_input_units, num_output_units],
dtype=dtype,
initializer=weight_init,
collections=weight_collections,
regularizer=weight_regularizer)
y = standard_ops.matmul(x, w)
if bias_init is not None:
b = _bias_variable(shape=[num_output_units],
dtype=dtype,
initializer=bias_init,
collections=bias_collections,
regularizer=bias_regularizer)
y = nn.bias_add(y, b)
return _apply_activation(y, activation_fn, output_collections)
def convolution2d(x,
num_output_channels,
kernel_size,
activation_fn=None,
stride=(1, 1),
padding='SAME',
weight_init=initializers.xavier_initializer_conv2d(),
bias_init=standard_ops.constant_initializer(0.),
name=None,
weight_collections=None,
bias_collections=None,
output_collections=None,
weight_regularizer=None,
bias_regularizer=None):
"""Adds the parameters for a conv2d layer and returns the output.
A neural network convolution layer is generally defined as:
\\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`,
**conv2d** is `tf.nn.conv2d` and `x` has shape
`[batch, height, width, channels]`. The output of this op is of shape
`[batch, out_height, out_width, num_output_channels]`, where `out_width` and
`out_height` are determined by the `padding` argument. See `conv2D` for
details.
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: A 4-D input `Tensor`.
num_output_channels: The number of output channels (i.e. the size of the
last dimension of the output).
kernel_size: A length 2 `list` or `tuple` containing the kernel size.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity.
stride: A length 2 `list` or `tuple` specifying the stride of the sliding
window across the image.
padding: A `string` from: "SAME", "VALID". The type of padding algorithm to
use.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "convolution2d" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections to which weights are added.
bias_collections: List of graph collections to which biases are added.
output_collections: List of graph collections to which outputs are added.
weight_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for weights.
bias_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for biases.
Returns:
The result of applying a 2-D convolutional layer.
Raises:
ValueError: If `kernel_size` or `stride` are not length 2.
"""
with variable_scope.variable_op_scope([x], name, 'convolution2d'):
num_input_channels = x.get_shape().dims[3].value
if len(kernel_size) != 2:
raise ValueError('kernel_size must be length 2: ' % kernel_size)
if len(stride) != 2:
raise ValueError('stride must be length 2: ' % kernel_size)
stride = [1, stride[0], stride[1], 1]
shape = [
kernel_size[0], kernel_size[1], num_input_channels,
num_output_channels
]
dtype = x.dtype.base_dtype
w = _weight_variable(shape=shape,
dtype=dtype,
initializer=weight_init,
collections=weight_collections,
regularizer=weight_regularizer)
y = nn.conv2d(x, w, stride, padding)
if bias_init is not None:
b = _bias_variable(shape=[num_output_channels],
dtype=dtype,
initializer=bias_init,
collections=bias_collections,
regularizer=bias_regularizer)
y = nn.bias_add(y, b)
return _apply_activation(y, activation_fn, output_collections)
def convolution2d(x,
num_output_channels,
kernel_size,
activation_fn=None,
stride=(1, 1),
padding='SAME',
weight_init=None,
bias_init=standard_ops.constant_initializer(0.),
num_input_channels=None,
name=None,
weight_collections=None,
bias_collections=None,
weight_regularizer=None,
create_summaries=True):
"""Adds the parameters for a conv2d layer and returns the output.
A neural network convolution layer is generally defined as:
\\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`,
**conv2d** is `nn.conv2d` and `x` has shape
`[batch, height, width, channels]`
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
In almost all cases, the input channels can be inferred from the shape
of `x`, but if it is unspecified or additional size checks are
desired, then `num_input_channels` can be specified.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: The input `Tensor`.
num_output_channels: The number of output channels (i.e. the size of
dim[3]).
kernel_size: A length 2 `list` or `tuple` containing the kernel size.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity.
stride: A length 2 `list` or `tuple` specifying the stride of the sliding
window across the image.
padding: A `string` from: "SAME", "VALID". The type of padding algorithm to
use.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
num_input_channels: The length of the channel dimension in the input.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "convolution2d" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections for just weights.
bias_collections: List of graph collections for just bias.
weight_regularizer: A regularizer like the result of
`tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`.
create_summaries: Set to false to disable summaries.
Returns:
The result of applying a fully connected layer.
Raises:
ValueError: if `x` is not rank 4; or `x`'s channel dimension is not known
and `num_input_channels` is not specified.
"""
with variable_scope.variable_op_scope([x], name, 'convolution2d') as vs:
# Check rank and if num_input_channels is specified, make sure it matches.
x.get_shape().assert_is_compatible_with([None, None, None,
num_input_channels])
if not num_input_channels:
if x.get_shape().dims is None or x.get_shape().dims[3].value is None:
raise ValueError(
'If x has an unknown channels dimension then num_input_channels '
'must be specified; shape: %s num_input_channels: %s'
% (x.get_shape(), num_input_channels))
else:
num_input_channels = x.get_shape().dims[3].value
# QQQ: Should we accept a scalar for a square convolution?
if len(kernel_size) != 2:
raise ValueError('kernel_size must be length 2: ' % kernel_size)
if len(stride) != 2:
raise ValueError('stride must be length 2: ' % kernel_size)
stride = [1, stride[0], stride[1], 1]
shape = [kernel_size[0], kernel_size[1], num_input_channels,
num_output_channels]
patch_size = kernel_size[0] * kernel_size[1]
weight_init = weight_init or xavier_initializer(
num_input_channels * patch_size, num_output_channels * patch_size)
dtype = x.dtype.base_dtype
w = variable_scope.get_variable('weights',
shape=shape,
dtype=dtype,
initializer=weight_init,
collections=weight_collections)
if not vs.reuse and create_summaries:
_add_histogram_summary(w)
y = nn.conv2d(x, w, stride, padding)
# Regularization is only applied to the weights and not bias.
if weight_regularizer:
_apply_regularization(w, weight_regularizer)
if bias_init:
b = _bias_variable(
num_output_channels, dtype, bias_init, bias_collections,
create_summaries)
y = nn.bias_add(y, b)
if create_summaries:
return _apply_activation_with_summaries(y, activation_fn)
if activation_fn:
y = activation_fn(y)
return y
def fully_connected(x,
num_output_nodes,
activation_fn=None,
weight_init=None,
bias_init=standard_ops.constant_initializer(0.),
num_input_nodes=None,
name=None,
weight_collections=None,
bias_collections=None,
weight_regularizer=None,
create_summaries=True):
"""Adds the parameters for a fully connected layer and returns the output.
A fully connected layer is generally defined as a matrix multiply:
\\\\(y = f(w * x + b)\\\\) where **f** is given by `activation_fn`
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
In almost all cases, the number of input nodes can be inferred from the shape
of `x`, but if it is unspecified or additional size checks are desired, then
`num_input_nodes` can be specified.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: The input `Tensor`. Must be 2D.
num_output_nodes: The size of the output.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity. If None is used, do not apply any activation.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
num_input_nodes: The number of input nodes.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "fully_connected" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections for just weights.
bias_collections: List of graph collections for just bias.
weight_regularizer: A regularizer like the result of
`tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`.
create_summaries: Set to false to disable summaries.
Returns:
The result of applying a fully connected layer.
Raises:
ValueError: if `x` is not rank 2; or `x`'s second dimension is not known
and `num_input_nodes` is not specified.
"""
with variable_scope.variable_op_scope([x], name, 'fully_connected'):
# Check rank and if num_input_nodes is specified, make sure it matches.
# TODO(wicke): This does not work with scalar inputs (shape [batch_size,])
# TODO(wicke): We'd have to encode the broadcasting rules here to be safe.
x.get_shape().assert_is_compatible_with([None, num_input_nodes])
if not num_input_nodes:
if x.get_shape().dims is None or x.get_shape().dims[1].value is None:
raise ValueError(
'If x has an unknown second dimension then num_input_nodes '
'must be specified; shape: %s num_input_nodes: %s'
% (x.get_shape(), num_input_nodes))
else:
num_input_nodes = x.get_shape().dims[1].value
dtype = x.dtype.base_dtype
# Regularization is only applied to the weights and not bias.
w = _weight_variable_2d(
num_input_nodes, num_output_nodes, dtype=dtype, init=weight_init,
collections=weight_collections, regularizer=weight_regularizer,
create_summaries=create_summaries)
y = standard_ops.matmul(x, w)
if bias_init is not None:
b = _bias_variable(
num_output_nodes, dtype=dtype, init=bias_init,
collections=bias_collections, create_summaries=create_summaries)
y = nn.bias_add(y, b)
if create_summaries:
return _apply_activation_with_summaries(y, activation_fn)
if activation_fn:
y = activation_fn(y)
return y
def convolution2d(x,
num_output_channels,
kernel_size,
activation_fn=None,
stride=(1, 1),
padding='SAME',
weight_init=None,
bias_init=standard_ops.constant_initializer(0.),
num_input_channels=None,
name=None,
weight_collections=None,
bias_collections=None,
weight_regularizer=None,
create_summaries=True):
"""Adds the parameters for a conv2d layer and returns the output.
A neural network convolution layer is generally defined as:
\\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`,
**conv2d** is `nn.conv2d` and `x` has shape
`[batch, height, width, channels]`
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
In almost all cases, the input channels can be inferred from the shape
of `x`, but if it is unspecified or additional size checks are
desired, then `num_input_channels` can be specified.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: The input `Tensor`.
num_output_channels: The number of output channels (i.e. the size of
dim[3]).
kernel_size: A length 2 `list` or `tuple` containing the kernel size.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity.
stride: A length 2 `list` or `tuple` specifying the stride of the sliding
window across the image.
padding: A `string` from: "SAME", "VALID". The type of padding algorithm to
use.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
num_input_channels: The length of the channel dimension in the input.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "convolution2d" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections for just weights.
bias_collections: List of graph collections for just bias.
weight_regularizer: A regularizer like the result of
`tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`.
create_summaries: Set to false to disable summaries.
Returns:
The result of applying a fully connected layer.
Raises:
ValueError: if `x` is not rank 4; or `x`'s channel dimension is not known
and `num_input_channels` is not specified.
"""
with variable_scope.variable_op_scope([x], name, 'convolution2d') as vs:
# Check rank and if num_input_channels is specified, make sure it matches.
x.get_shape().assert_is_compatible_with(
[None, None, None, num_input_channels])
if not num_input_channels:
if x.get_shape().dims is None or x.get_shape(
).dims[3].value is None:
raise ValueError(
'If x has an unknown channels dimension then num_input_channels '
'must be specified; shape: %s num_input_channels: %s' %
(x.get_shape(), num_input_channels))
else:
num_input_channels = x.get_shape().dims[3].value
# QQQ: Should we accept a scalar for a square convolution?
if len(kernel_size) != 2:
raise ValueError('kernel_size must be length 2: ' % kernel_size)
if len(stride) != 2:
raise ValueError('stride must be length 2: ' % kernel_size)
stride = [1, stride[0], stride[1], 1]
shape = [
kernel_size[0], kernel_size[1], num_input_channels,
num_output_channels
]
patch_size = kernel_size[0] * kernel_size[1]
weight_init = weight_init or xavier_initializer(
num_input_channels * patch_size, num_output_channels * patch_size)
dtype = x.dtype.base_dtype
w = variable_scope.get_variable('weights',
shape=shape,
dtype=dtype,
initializer=weight_init,
collections=weight_collections)
if not vs.reuse and create_summaries:
_add_histogram_summary(w)
y = nn.conv2d(x, w, stride, padding)
# Regularization is only applied to the weights and not bias.
if weight_regularizer:
_apply_regularization(w, weight_regularizer)
if bias_init:
b = _bias_variable(num_output_channels, dtype, bias_init,
bias_collections, create_summaries)
y = nn.bias_add(y, b)
if create_summaries:
return _apply_activation_with_summaries(y, activation_fn)
if activation_fn:
y = activation_fn(y)
return y
def fully_connected(x,
num_output_nodes,
activation_fn=None,
weight_init=None,
bias_init=standard_ops.constant_initializer(0.),
num_input_nodes=None,
name=None,
weight_collections=None,
bias_collections=None,
weight_regularizer=None,
create_summaries=True):
"""Adds the parameters for a fully connected layer and returns the output.
A fully connected layer is generally defined as a matrix multiply:
\\\\(y = f(w * x + b)\\\\) where **f** is given by `activation_fn`
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
In almost all cases, the number of input nodes can be inferred from the shape
of `x`, but if it is unspecified or additional size checks are desired, then
`num_input_nodes` can be specified.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: The input `Tensor`. Must be 2D.
num_output_nodes: The size of the output.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity. If None is used, do not apply any activation.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
num_input_nodes: The number of input nodes.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "fully_connected" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections for just weights.
bias_collections: List of graph collections for just bias.
weight_regularizer: A regularizer like the result of
`tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`.
create_summaries: Set to false to disable summaries.
Returns:
The result of applying a fully connected layer.
Raises:
ValueError: if `x` is not rank 2; or `x`'s second dimension is not known
and `num_input_nodes` is not specified.
"""
with variable_scope.variable_op_scope([x], name, 'fully_connected'):
# Check rank and if num_input_nodes is specified, make sure it matches.
# TODO(wicke): This does not work with scalar inputs (shape [batch_size,])
# TODO(wicke): We'd have to encode the broadcasting rules here to be safe.
x.get_shape().assert_is_compatible_with([None, num_input_nodes])
if not num_input_nodes:
if x.get_shape().dims is None or x.get_shape(
).dims[1].value is None:
raise ValueError(
'If x has an unknown second dimension then num_input_nodes '
'must be specified; shape: %s num_input_nodes: %s' %
(x.get_shape(), num_input_nodes))
else:
num_input_nodes = x.get_shape().dims[1].value
dtype = x.dtype.base_dtype
# Regularization is only applied to the weights and not bias.
w = _weight_variable_2d(num_input_nodes,
num_output_nodes,
dtype=dtype,
init=weight_init,
collections=weight_collections,
regularizer=weight_regularizer,
create_summaries=create_summaries)
y = standard_ops.matmul(x, w)
if bias_init is not None:
b = _bias_variable(num_output_nodes,
dtype=dtype,
init=bias_init,
collections=bias_collections,
create_summaries=create_summaries)
y = nn.bias_add(y, b)
if create_summaries:
return _apply_activation_with_summaries(y, activation_fn)
if activation_fn:
y = activation_fn(y)
return y
def fully_connected(x,
num_output_units,
activation_fn=None,
weight_init=initializers.xavier_initializer(),
bias_init=standard_ops.constant_initializer(0.),
name=None,
weight_collections=(ops.GraphKeys.WEIGHTS,),
bias_collections=(ops.GraphKeys.BIASES,),
output_collections=(ops.GraphKeys.ACTIVATIONS,),
weight_regularizer=None,
bias_regularizer=None):
"""Adds the parameters for a fully connected layer and returns the output.
A fully connected layer is generally defined as a matrix multiply:
`y = f(w * x + b)` where `f` is given by `activation_fn`. If
`activation_fn` is `None`, the result of `y = w * x + b` is
returned.
This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting
`bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which in collections to place
the created variables (`weight_collections` and `bias_collections`; note that
the variables are always added to the `VARIABLES` collection). The output of
the layer can be placed in custom collections using `output_collections`.
The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`,
respectively.
A per layer regularization can be specified by setting `weight_regularizer`
and `bias_regularizer`, which are applied to the weights and biases
respectively, and whose output is added to the `REGULARIZATION_LOSSES`
collection.
Args:
x: The input `Tensor`.
num_output_units: The size of the output.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity. If None is used, do not apply any activation.
weight_init: An optional weight initialization, defaults to
`xavier_initializer`.
bias_init: An initializer for the bias, defaults to 0. Set to `None` in
order to disable bias.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "fully_connected" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections to which weights are added.
bias_collections: List of graph collections to which biases are added.
output_collections: List of graph collections to which outputs are added.
weight_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for weights.
bias_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for biases.
Returns:
The output of the fully connected layer.
"""
with variable_scope.variable_op_scope([x], name, 'fully_connected'):
num_input_units = x.get_shape().dims[1].value
dtype = x.dtype.base_dtype
w = _weight_variable(shape=[num_input_units, num_output_units],
dtype=dtype,
initializer=weight_init,
collections=weight_collections,
regularizer=weight_regularizer)
y = standard_ops.matmul(x, w)
if bias_init is not None:
b = _bias_variable(shape=[num_output_units],
dtype=dtype,
initializer=bias_init,
collections=bias_collections,
regularizer=bias_regularizer)
y = nn.bias_add(y, b)
return _apply_activation(y, activation_fn, output_collections)
def convolution2d(x,
num_output_channels,
kernel_size,
activation_fn=None,
stride=(1, 1),
padding='SAME',
weight_init=initializers.xavier_initializer_conv2d(),
bias_init=standard_ops.constant_initializer(0.),
name=None,
weight_collections=None,
bias_collections=None,
output_collections=None,
weight_regularizer=None,
bias_regularizer=None):
"""Adds the parameters for a conv2d layer and returns the output.
A neural network convolution layer is generally defined as:
\\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`,
**conv2d** is `tf.nn.conv2d` and `x` has shape
`[batch, height, width, channels]`. The output of this op is of shape
`[batch, out_height, out_width, num_output_channels]`, where `out_width` and
`out_height` are determined by the `padding` argument. See `conv2D` for
details.
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: A 4-D input `Tensor`.
num_output_channels: The number of output channels (i.e. the size of the
last dimension of the output).
kernel_size: A length 2 `list` or `tuple` containing the kernel size.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity.
stride: A length 2 `list` or `tuple` specifying the stride of the sliding
window across the image.
padding: A `string` from: "SAME", "VALID". The type of padding algorithm to
use.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "convolution2d" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections to which weights are added.
bias_collections: List of graph collections to which biases are added.
output_collections: List of graph collections to which outputs are added.
weight_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for weights.
bias_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for biases.
Returns:
The result of applying a 2-D convolutional layer.
Raises:
ValueError: If `kernel_size` or `stride` are not length 2.
"""
with variable_scope.variable_op_scope([x], name, 'convolution2d'):
num_input_channels = x.get_shape().dims[3].value
if len(kernel_size) != 2:
raise ValueError('kernel_size must be length 2: ' % kernel_size)
if len(stride) != 2:
raise ValueError('stride must be length 2: ' % kernel_size)
stride = [1, stride[0], stride[1], 1]
shape = [kernel_size[0], kernel_size[1], num_input_channels,
num_output_channels]
dtype = x.dtype.base_dtype
w = _weight_variable(shape=shape,
dtype=dtype,
initializer=weight_init,
collections=weight_collections,
regularizer=weight_regularizer)
y = nn.conv2d(x, w, stride, padding)
if bias_init is not None:
b = _bias_variable(shape=[num_output_channels],
dtype=dtype,
initializer=bias_init,
collections=bias_collections,
regularizer=bias_regularizer)
y = nn.bias_add(y, b)
return _apply_activation(y, activation_fn, output_collections)
def fully_connected(x,
num_output_units,
activation_fn=None,
weight_init=initializers.xavier_initializer(),
bias_init=standard_ops.constant_initializer(0.),
name=None,
weight_collections=(ops.GraphKeys.WEIGHTS,),
bias_collections=(ops.GraphKeys.BIASES,),
output_collections=(ops.GraphKeys.ACTIVATIONS,),
weight_regularizer=None,
bias_regularizer=None):
# pylint: disable=anomalous-backslash-in-string
r"""Adds the parameters for a fully connected layer and returns the output.
A fully connected layer is generally defined as a matrix multiply:
`y = f(w * x + b)` where `f` is given by `activation_fn`. If
`activation_fn` is `None`, the result of `y = w * x + b` is
returned.
If `x` has shape [\\\(\\text{dim}_0, \\text{dim}_1, ..., \\text{dim}_n\\\)]
with more than 2 dimensions (\\\(n > 1\\\)), then we repeat the matrix
multiply along the first dimensions. The result r is a tensor of shape
[\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`],
where \\\( r_{i_0, ..., i_{n-1}, k} =
\\sum_{0 \\leq j < \\text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\\).
This is accomplished by reshaping `x` to 2-D
[\\\(\\text{dim}_0 \\cdot ... \\cdot \\text{dim}_{n-1}, \\text{dim}_n\\\)]
before the matrix multiply and afterwards reshaping it to
[\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`].
This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting
`bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which in collections to place
the created variables (`weight_collections` and `bias_collections`; note that
the variables are always added to the `VARIABLES` collection). The output of
the layer can be placed in custom collections using `output_collections`.
The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`,
respectively.
A per layer regularization can be specified by setting `weight_regularizer`
and `bias_regularizer`, which are applied to the weights and biases
respectively, and whose output is added to the `REGULARIZATION_LOSSES`
collection.
Args:
x: The input `Tensor`.
num_output_units: The size of the output.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity. If None is used, do not apply any activation.
weight_init: An optional weight initialization, defaults to
`xavier_initializer`.
bias_init: An initializer for the bias, defaults to 0. Set to `None` in
order to disable bias.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "fully_connected" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections to which weights are added.
bias_collections: List of graph collections to which biases are added.
output_collections: List of graph collections to which outputs are added.
weight_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for weights.
bias_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for biases.
Returns:
The output of the fully connected layer.
Raises:
ValueError: if x has rank less than 2 or if its last dimension is not set.
"""
# pylint: enable=anomalous-backslash-in-string
with variable_scope.variable_op_scope([x], name, 'fully_connected'):
dims = x.get_shape().dims
if dims is None:
raise ValueError('dims of x must be known but is None')
if len(dims) < 2:
raise ValueError('rank of x must be at least 2 not: %d' % len(dims))
num_input_units = dims[-1].value
if num_input_units is None:
raise ValueError('last dimension of x must be known but is None')
dtype = x.dtype.base_dtype
weight_collections = set(list(weight_collections or []) +
[ops.GraphKeys.VARIABLES])
w = variable_scope.get_variable('weights',
shape=[num_input_units, num_output_units],
dtype=dtype,
initializer=weight_init,
collections=weight_collections,
regularizer=weight_regularizer)
x_2_dim = x if len(dims) <= 2 else array_ops.reshape(x,
[-1, num_input_units])
y = standard_ops.matmul(x_2_dim, w)
if bias_init is not None:
bias_collections = set(list(bias_collections or []) +
[ops.GraphKeys.VARIABLES])
b = variable_scope.get_variable('bias',
shape=[num_output_units],
dtype=dtype,
initializer=bias_init,
collections=bias_collections,
regularizer=bias_regularizer)
y = nn.bias_add(y, b)
if len(dims) > 2:
out_shape = array_ops.unpack(array_ops.shape(x))
out_shape[-1] = num_output_units
y = array_ops.reshape(y, array_ops.pack(out_shape))
static_shape = x.get_shape().as_list()
static_shape[-1] = num_output_units
y.set_shape(static_shape)
return _apply_activation(y, activation_fn, output_collections)
def fully_connected(x,
num_output_units,
activation_fn=None,
weight_init=initializers.xavier_initializer(),
bias_init=standard_ops.constant_initializer(0.),
name=None,
weight_collections=(ops.GraphKeys.WEIGHTS,),
bias_collections=(ops.GraphKeys.BIASES,),
output_collections=(ops.GraphKeys.ACTIVATIONS,),
trainable=True,
weight_regularizer=None,
bias_regularizer=None):
# pylint: disable=anomalous-backslash-in-string
r"""Adds the parameters for a fully connected layer and returns the output.
A fully connected layer is generally defined as a matrix multiply:
`y = f(w * x + b)` where `f` is given by `activation_fn`. If
`activation_fn` is `None`, the result of `y = w * x + b` is
returned.
If `x` has shape [\\\(\\text{dim}_0, \\text{dim}_1, ..., \\text{dim}_n\\\)]
with more than 2 dimensions (\\\(n > 1\\\)), then we repeat the matrix
multiply along the first dimensions. The result r is a tensor of shape
[\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`],
where \\\( r_{i_0, ..., i_{n-1}, k} =
\\sum_{0 \\leq j < \\text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\\).
This is accomplished by reshaping `x` to 2-D
[\\\(\\text{dim}_0 \\cdot ... \\cdot \\text{dim}_{n-1}, \\text{dim}_n\\\)]
before the matrix multiply and afterwards reshaping it to
[\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`].
This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting
`bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which in collections to place
the created variables (`weight_collections` and `bias_collections`; note that
the variables are always added to the `VARIABLES` collection). The output of
the layer can be placed in custom collections using `output_collections`.
The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`,
respectively.
A per layer regularization can be specified by setting `weight_regularizer`
and `bias_regularizer`, which are applied to the weights and biases
respectively, and whose output is added to the `REGULARIZATION_LOSSES`
collection.
Args:
x: The input `Tensor`.
num_output_units: The size of the output.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity. If None is used, do not apply any activation.
weight_init: An optional weight initialization, defaults to
`xavier_initializer`.
bias_init: An initializer for the bias, defaults to 0. Set to `None` in
order to disable bias.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "fully_connected" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections to which weights are added.
bias_collections: List of graph collections to which biases are added.
output_collections: List of graph collections to which outputs are added.
trainable: If `True` also add variables to the graph collection
`GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
weight_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for weights.
bias_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for biases.
Returns:
The output of the fully connected layer.
Raises:
ValueError: if x has rank less than 2 or if its last dimension is not set.
"""
# pylint: enable=anomalous-backslash-in-string
with variable_scope.variable_op_scope([x], name, 'fully_connected'):
dims = x.get_shape().dims
if dims is None:
raise ValueError('dims of x must be known but is None')
if len(dims) < 2:
raise ValueError('rank of x must be at least 2 not: %d' % len(dims))
num_input_units = dims[-1].value
if num_input_units is None:
raise ValueError('last dimension of x must be known but is None')
dtype = x.dtype.base_dtype
weight_collections = set(list(weight_collections or []) +
[ops.GraphKeys.VARIABLES])
w = variable_scope.get_variable('weights',
shape=[num_input_units, num_output_units],
dtype=dtype,
initializer=weight_init,
collections=weight_collections,
regularizer=weight_regularizer,
trainable=trainable)
x_2_dim = x if len(dims) <= 2 else array_ops.reshape(x,
[-1, num_input_units])
y = standard_ops.matmul(x_2_dim, w)
if bias_init is not None:
bias_collections = set(list(bias_collections or []) +
[ops.GraphKeys.VARIABLES])
b = variable_scope.get_variable('bias',
shape=[num_output_units],
dtype=dtype,
initializer=bias_init,
collections=bias_collections,
regularizer=bias_regularizer,
trainable=trainable)
y = nn.bias_add(y, b)
if len(dims) > 2:
out_shape = array_ops.unpack(array_ops.shape(x))
out_shape[-1] = num_output_units
y = array_ops.reshape(y, array_ops.pack(out_shape))
static_shape = x.get_shape().as_list()
static_shape[-1] = num_output_units
y.set_shape(static_shape)
return _apply_activation(y, activation_fn, output_collections)
