def _bias_variable(num_output_nodes, dtype, init=standard_ops.constant_initializer(0.), collections=None, create_summaries=True): """Create bias variable with shape `(num_output_nodes,)`. Args: num_output_nodes: The size of the output. dtype: Data type of the variable. init: An optional initialization. If not specified, use zeros. collections: List of graph collections to which we'll add the new variable. create_summaries: Set to false to disable summaries. Returns: The result weights variable. """ b = variable_scope.get_variable('bias', shape=(num_output_nodes, ), dtype=dtype, initializer=init, collections=collections) if not variable_scope.get_variable_scope().reuse and create_summaries: _add_histogram_summary(b) return b
def _bias_variable( num_output_nodes, dtype, init=standard_ops.constant_initializer(0.), collections=None, create_summaries=True): """Create bias variable with shape `(num_output_nodes,)`. Args: num_output_nodes: The size of the output. dtype: Data type of the variable. init: An optional initialization. If not specified, use zeros. collections: List of graph collections to which we'll add the new variable. create_summaries: Set to false to disable summaries. Returns: The result weights variable. """ b = variable_scope.get_variable( 'bias', shape=(num_output_nodes,), dtype=dtype, initializer=init, collections=collections) if not variable_scope.get_variable_scope().reuse and create_summaries: _add_histogram_summary(b) return b
def fully_connected(x, num_output_units, activation_fn=None, weight_init=initializers.xavier_initializer(), bias_init=standard_ops.constant_initializer(0.), name=None, weight_collections=(ops.GraphKeys.WEIGHTS, ), bias_collections=(ops.GraphKeys.BIASES, ), output_collections=(ops.GraphKeys.ACTIVATIONS, ), weight_regularizer=None, bias_regularizer=None): """Adds the parameters for a fully connected layer and returns the output. A fully connected layer is generally defined as a matrix multiply: `y = f(w * x + b)` where `f` is given by `activation_fn`. If `activation_fn` is `None`, the result of `y = w * x + b` is returned. This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which in collections to place the created variables (`weight_collections` and `bias_collections`; note that the variables are always added to the `VARIABLES` collection). The output of the layer can be placed in custom collections using `output_collections`. The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`, respectively. A per layer regularization can be specified by setting `weight_regularizer` and `bias_regularizer`, which are applied to the weights and biases respectively, and whose output is added to the `REGULARIZATION_LOSSES` collection. Args: x: The input `Tensor`. num_output_units: The size of the output. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. If None is used, do not apply any activation. weight_init: An optional weight initialization, defaults to `xavier_initializer`. bias_init: An initializer for the bias, defaults to 0. Set to `None` in order to disable bias. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "fully_connected" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections to which weights are added. bias_collections: List of graph collections to which biases are added. output_collections: List of graph collections to which outputs are added. weight_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for weights. bias_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for biases. Returns: The output of the fully connected layer. """ with variable_scope.variable_op_scope([x], name, 'fully_connected'): num_input_units = x.get_shape().dims[1].value dtype = x.dtype.base_dtype w = _weight_variable(shape=[num_input_units, num_output_units], dtype=dtype, initializer=weight_init, collections=weight_collections, regularizer=weight_regularizer) y = standard_ops.matmul(x, w) if bias_init is not None: b = _bias_variable(shape=[num_output_units], dtype=dtype, initializer=bias_init, collections=bias_collections, regularizer=bias_regularizer) y = nn.bias_add(y, b) return _apply_activation(y, activation_fn, output_collections)
def convolution2d(x, num_output_channels, kernel_size, activation_fn=None, stride=(1, 1), padding='SAME', weight_init=initializers.xavier_initializer_conv2d(), bias_init=standard_ops.constant_initializer(0.), name=None, weight_collections=None, bias_collections=None, output_collections=None, weight_regularizer=None, bias_regularizer=None): """Adds the parameters for a conv2d layer and returns the output. A neural network convolution layer is generally defined as: \\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`, **conv2d** is `tf.nn.conv2d` and `x` has shape `[batch, height, width, channels]`. The output of this op is of shape `[batch, out_height, out_width, num_output_channels]`, where `out_width` and `out_height` are determined by the `padding` argument. See `conv2D` for details. This op creates `w` and optionally `b` and adds various summaries that can be useful for visualizing learning or diagnosing training problems. Bias can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which collections to place the created variables in (`weight_collections` and `bias_collections`). A per layer regularization can be specified by setting `weight_regularizer`. This is only applied to weights and not the bias. Args: x: A 4-D input `Tensor`. num_output_channels: The number of output channels (i.e. the size of the last dimension of the output). kernel_size: A length 2 `list` or `tuple` containing the kernel size. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. stride: A length 2 `list` or `tuple` specifying the stride of the sliding window across the image. padding: A `string` from: "SAME", "VALID". The type of padding algorithm to use. weight_init: An optional initialization. If not specified, uses Xavier initialization (see `tf.learn.xavier_initializer`). bias_init: An initializer for the bias, defaults to 0. Set to`None` in order to disable bias. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "convolution2d" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections to which weights are added. bias_collections: List of graph collections to which biases are added. output_collections: List of graph collections to which outputs are added. weight_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for weights. bias_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for biases. Returns: The result of applying a 2-D convolutional layer. Raises: ValueError: If `kernel_size` or `stride` are not length 2. """ with variable_scope.variable_op_scope([x], name, 'convolution2d'): num_input_channels = x.get_shape().dims[3].value if len(kernel_size) != 2: raise ValueError('kernel_size must be length 2: ' % kernel_size) if len(stride) != 2: raise ValueError('stride must be length 2: ' % kernel_size) stride = [1, stride[0], stride[1], 1] shape = [ kernel_size[0], kernel_size[1], num_input_channels, num_output_channels ] dtype = x.dtype.base_dtype w = _weight_variable(shape=shape, dtype=dtype, initializer=weight_init, collections=weight_collections, regularizer=weight_regularizer) y = nn.conv2d(x, w, stride, padding) if bias_init is not None: b = _bias_variable(shape=[num_output_channels], dtype=dtype, initializer=bias_init, collections=bias_collections, regularizer=bias_regularizer) y = nn.bias_add(y, b) return _apply_activation(y, activation_fn, output_collections)
def convolution2d(x, num_output_channels, kernel_size, activation_fn=None, stride=(1, 1), padding='SAME', weight_init=None, bias_init=standard_ops.constant_initializer(0.), num_input_channels=None, name=None, weight_collections=None, bias_collections=None, weight_regularizer=None, create_summaries=True): """Adds the parameters for a conv2d layer and returns the output. A neural network convolution layer is generally defined as: \\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`, **conv2d** is `nn.conv2d` and `x` has shape `[batch, height, width, channels]` This op creates `w` and optionally `b` and adds various summaries that can be useful for visualizing learning or diagnosing training problems. Bias can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. In almost all cases, the input channels can be inferred from the shape of `x`, but if it is unspecified or additional size checks are desired, then `num_input_channels` can be specified. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which collections to place the created variables in (`weight_collections` and `bias_collections`). A per layer regularization can be specified by setting `weight_regularizer`. This is only applied to weights and not the bias. Args: x: The input `Tensor`. num_output_channels: The number of output channels (i.e. the size of dim[3]). kernel_size: A length 2 `list` or `tuple` containing the kernel size. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. stride: A length 2 `list` or `tuple` specifying the stride of the sliding window across the image. padding: A `string` from: "SAME", "VALID". The type of padding algorithm to use. weight_init: An optional initialization. If not specified, uses Xavier initialization (see `tf.learn.xavier_initializer`). bias_init: An initializer for the bias, defaults to 0. Set to`None` in order to disable bias. num_input_channels: The length of the channel dimension in the input. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "convolution2d" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections for just weights. bias_collections: List of graph collections for just bias. weight_regularizer: A regularizer like the result of `tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`. create_summaries: Set to false to disable summaries. Returns: The result of applying a fully connected layer. Raises: ValueError: if `x` is not rank 4; or `x`'s channel dimension is not known and `num_input_channels` is not specified. """ with variable_scope.variable_op_scope([x], name, 'convolution2d') as vs: # Check rank and if num_input_channels is specified, make sure it matches. x.get_shape().assert_is_compatible_with([None, None, None, num_input_channels]) if not num_input_channels: if x.get_shape().dims is None or x.get_shape().dims[3].value is None: raise ValueError( 'If x has an unknown channels dimension then num_input_channels ' 'must be specified; shape: %s num_input_channels: %s' % (x.get_shape(), num_input_channels)) else: num_input_channels = x.get_shape().dims[3].value # QQQ: Should we accept a scalar for a square convolution? if len(kernel_size) != 2: raise ValueError('kernel_size must be length 2: ' % kernel_size) if len(stride) != 2: raise ValueError('stride must be length 2: ' % kernel_size) stride = [1, stride[0], stride[1], 1] shape = [kernel_size[0], kernel_size[1], num_input_channels, num_output_channels] patch_size = kernel_size[0] * kernel_size[1] weight_init = weight_init or xavier_initializer( num_input_channels * patch_size, num_output_channels * patch_size) dtype = x.dtype.base_dtype w = variable_scope.get_variable('weights', shape=shape, dtype=dtype, initializer=weight_init, collections=weight_collections) if not vs.reuse and create_summaries: _add_histogram_summary(w) y = nn.conv2d(x, w, stride, padding) # Regularization is only applied to the weights and not bias. if weight_regularizer: _apply_regularization(w, weight_regularizer) if bias_init: b = _bias_variable( num_output_channels, dtype, bias_init, bias_collections, create_summaries) y = nn.bias_add(y, b) if create_summaries: return _apply_activation_with_summaries(y, activation_fn) if activation_fn: y = activation_fn(y) return y
def fully_connected(x, num_output_nodes, activation_fn=None, weight_init=None, bias_init=standard_ops.constant_initializer(0.), num_input_nodes=None, name=None, weight_collections=None, bias_collections=None, weight_regularizer=None, create_summaries=True): """Adds the parameters for a fully connected layer and returns the output. A fully connected layer is generally defined as a matrix multiply: \\\\(y = f(w * x + b)\\\\) where **f** is given by `activation_fn` This op creates `w` and optionally `b` and adds various summaries that can be useful for visualizing learning or diagnosing training problems. Bias can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. In almost all cases, the number of input nodes can be inferred from the shape of `x`, but if it is unspecified or additional size checks are desired, then `num_input_nodes` can be specified. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which collections to place the created variables in (`weight_collections` and `bias_collections`). A per layer regularization can be specified by setting `weight_regularizer`. This is only applied to weights and not the bias. Args: x: The input `Tensor`. Must be 2D. num_output_nodes: The size of the output. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. If None is used, do not apply any activation. weight_init: An optional initialization. If not specified, uses Xavier initialization (see `tf.learn.xavier_initializer`). bias_init: An initializer for the bias, defaults to 0. Set to`None` in order to disable bias. num_input_nodes: The number of input nodes. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "fully_connected" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections for just weights. bias_collections: List of graph collections for just bias. weight_regularizer: A regularizer like the result of `tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`. create_summaries: Set to false to disable summaries. Returns: The result of applying a fully connected layer. Raises: ValueError: if `x` is not rank 2; or `x`'s second dimension is not known and `num_input_nodes` is not specified. """ with variable_scope.variable_op_scope([x], name, 'fully_connected'): # Check rank and if num_input_nodes is specified, make sure it matches. # TODO(wicke): This does not work with scalar inputs (shape [batch_size,]) # TODO(wicke): We'd have to encode the broadcasting rules here to be safe. x.get_shape().assert_is_compatible_with([None, num_input_nodes]) if not num_input_nodes: if x.get_shape().dims is None or x.get_shape().dims[1].value is None: raise ValueError( 'If x has an unknown second dimension then num_input_nodes ' 'must be specified; shape: %s num_input_nodes: %s' % (x.get_shape(), num_input_nodes)) else: num_input_nodes = x.get_shape().dims[1].value dtype = x.dtype.base_dtype # Regularization is only applied to the weights and not bias. w = _weight_variable_2d( num_input_nodes, num_output_nodes, dtype=dtype, init=weight_init, collections=weight_collections, regularizer=weight_regularizer, create_summaries=create_summaries) y = standard_ops.matmul(x, w) if bias_init is not None: b = _bias_variable( num_output_nodes, dtype=dtype, init=bias_init, collections=bias_collections, create_summaries=create_summaries) y = nn.bias_add(y, b) if create_summaries: return _apply_activation_with_summaries(y, activation_fn) if activation_fn: y = activation_fn(y) return y
def convolution2d(x, num_output_channels, kernel_size, activation_fn=None, stride=(1, 1), padding='SAME', weight_init=None, bias_init=standard_ops.constant_initializer(0.), num_input_channels=None, name=None, weight_collections=None, bias_collections=None, weight_regularizer=None, create_summaries=True): """Adds the parameters for a conv2d layer and returns the output. A neural network convolution layer is generally defined as: \\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`, **conv2d** is `nn.conv2d` and `x` has shape `[batch, height, width, channels]` This op creates `w` and optionally `b` and adds various summaries that can be useful for visualizing learning or diagnosing training problems. Bias can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. In almost all cases, the input channels can be inferred from the shape of `x`, but if it is unspecified or additional size checks are desired, then `num_input_channels` can be specified. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which collections to place the created variables in (`weight_collections` and `bias_collections`). A per layer regularization can be specified by setting `weight_regularizer`. This is only applied to weights and not the bias. Args: x: The input `Tensor`. num_output_channels: The number of output channels (i.e. the size of dim[3]). kernel_size: A length 2 `list` or `tuple` containing the kernel size. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. stride: A length 2 `list` or `tuple` specifying the stride of the sliding window across the image. padding: A `string` from: "SAME", "VALID". The type of padding algorithm to use. weight_init: An optional initialization. If not specified, uses Xavier initialization (see `tf.learn.xavier_initializer`). bias_init: An initializer for the bias, defaults to 0. Set to`None` in order to disable bias. num_input_channels: The length of the channel dimension in the input. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "convolution2d" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections for just weights. bias_collections: List of graph collections for just bias. weight_regularizer: A regularizer like the result of `tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`. create_summaries: Set to false to disable summaries. Returns: The result of applying a fully connected layer. Raises: ValueError: if `x` is not rank 4; or `x`'s channel dimension is not known and `num_input_channels` is not specified. """ with variable_scope.variable_op_scope([x], name, 'convolution2d') as vs: # Check rank and if num_input_channels is specified, make sure it matches. x.get_shape().assert_is_compatible_with( [None, None, None, num_input_channels]) if not num_input_channels: if x.get_shape().dims is None or x.get_shape( ).dims[3].value is None: raise ValueError( 'If x has an unknown channels dimension then num_input_channels ' 'must be specified; shape: %s num_input_channels: %s' % (x.get_shape(), num_input_channels)) else: num_input_channels = x.get_shape().dims[3].value # QQQ: Should we accept a scalar for a square convolution? if len(kernel_size) != 2: raise ValueError('kernel_size must be length 2: ' % kernel_size) if len(stride) != 2: raise ValueError('stride must be length 2: ' % kernel_size) stride = [1, stride[0], stride[1], 1] shape = [ kernel_size[0], kernel_size[1], num_input_channels, num_output_channels ] patch_size = kernel_size[0] * kernel_size[1] weight_init = weight_init or xavier_initializer( num_input_channels * patch_size, num_output_channels * patch_size) dtype = x.dtype.base_dtype w = variable_scope.get_variable('weights', shape=shape, dtype=dtype, initializer=weight_init, collections=weight_collections) if not vs.reuse and create_summaries: _add_histogram_summary(w) y = nn.conv2d(x, w, stride, padding) # Regularization is only applied to the weights and not bias. if weight_regularizer: _apply_regularization(w, weight_regularizer) if bias_init: b = _bias_variable(num_output_channels, dtype, bias_init, bias_collections, create_summaries) y = nn.bias_add(y, b) if create_summaries: return _apply_activation_with_summaries(y, activation_fn) if activation_fn: y = activation_fn(y) return y
def fully_connected(x, num_output_nodes, activation_fn=None, weight_init=None, bias_init=standard_ops.constant_initializer(0.), num_input_nodes=None, name=None, weight_collections=None, bias_collections=None, weight_regularizer=None, create_summaries=True): """Adds the parameters for a fully connected layer and returns the output. A fully connected layer is generally defined as a matrix multiply: \\\\(y = f(w * x + b)\\\\) where **f** is given by `activation_fn` This op creates `w` and optionally `b` and adds various summaries that can be useful for visualizing learning or diagnosing training problems. Bias can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. In almost all cases, the number of input nodes can be inferred from the shape of `x`, but if it is unspecified or additional size checks are desired, then `num_input_nodes` can be specified. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which collections to place the created variables in (`weight_collections` and `bias_collections`). A per layer regularization can be specified by setting `weight_regularizer`. This is only applied to weights and not the bias. Args: x: The input `Tensor`. Must be 2D. num_output_nodes: The size of the output. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. If None is used, do not apply any activation. weight_init: An optional initialization. If not specified, uses Xavier initialization (see `tf.learn.xavier_initializer`). bias_init: An initializer for the bias, defaults to 0. Set to`None` in order to disable bias. num_input_nodes: The number of input nodes. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "fully_connected" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections for just weights. bias_collections: List of graph collections for just bias. weight_regularizer: A regularizer like the result of `tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`. create_summaries: Set to false to disable summaries. Returns: The result of applying a fully connected layer. Raises: ValueError: if `x` is not rank 2; or `x`'s second dimension is not known and `num_input_nodes` is not specified. """ with variable_scope.variable_op_scope([x], name, 'fully_connected'): # Check rank and if num_input_nodes is specified, make sure it matches. # TODO(wicke): This does not work with scalar inputs (shape [batch_size,]) # TODO(wicke): We'd have to encode the broadcasting rules here to be safe. x.get_shape().assert_is_compatible_with([None, num_input_nodes]) if not num_input_nodes: if x.get_shape().dims is None or x.get_shape( ).dims[1].value is None: raise ValueError( 'If x has an unknown second dimension then num_input_nodes ' 'must be specified; shape: %s num_input_nodes: %s' % (x.get_shape(), num_input_nodes)) else: num_input_nodes = x.get_shape().dims[1].value dtype = x.dtype.base_dtype # Regularization is only applied to the weights and not bias. w = _weight_variable_2d(num_input_nodes, num_output_nodes, dtype=dtype, init=weight_init, collections=weight_collections, regularizer=weight_regularizer, create_summaries=create_summaries) y = standard_ops.matmul(x, w) if bias_init is not None: b = _bias_variable(num_output_nodes, dtype=dtype, init=bias_init, collections=bias_collections, create_summaries=create_summaries) y = nn.bias_add(y, b) if create_summaries: return _apply_activation_with_summaries(y, activation_fn) if activation_fn: y = activation_fn(y) return y
def fully_connected(x, num_output_units, activation_fn=None, weight_init=initializers.xavier_initializer(), bias_init=standard_ops.constant_initializer(0.), name=None, weight_collections=(ops.GraphKeys.WEIGHTS,), bias_collections=(ops.GraphKeys.BIASES,), output_collections=(ops.GraphKeys.ACTIVATIONS,), weight_regularizer=None, bias_regularizer=None): """Adds the parameters for a fully connected layer and returns the output. A fully connected layer is generally defined as a matrix multiply: `y = f(w * x + b)` where `f` is given by `activation_fn`. If `activation_fn` is `None`, the result of `y = w * x + b` is returned. This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which in collections to place the created variables (`weight_collections` and `bias_collections`; note that the variables are always added to the `VARIABLES` collection). The output of the layer can be placed in custom collections using `output_collections`. The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`, respectively. A per layer regularization can be specified by setting `weight_regularizer` and `bias_regularizer`, which are applied to the weights and biases respectively, and whose output is added to the `REGULARIZATION_LOSSES` collection. Args: x: The input `Tensor`. num_output_units: The size of the output. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. If None is used, do not apply any activation. weight_init: An optional weight initialization, defaults to `xavier_initializer`. bias_init: An initializer for the bias, defaults to 0. Set to `None` in order to disable bias. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "fully_connected" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections to which weights are added. bias_collections: List of graph collections to which biases are added. output_collections: List of graph collections to which outputs are added. weight_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for weights. bias_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for biases. Returns: The output of the fully connected layer. """ with variable_scope.variable_op_scope([x], name, 'fully_connected'): num_input_units = x.get_shape().dims[1].value dtype = x.dtype.base_dtype w = _weight_variable(shape=[num_input_units, num_output_units], dtype=dtype, initializer=weight_init, collections=weight_collections, regularizer=weight_regularizer) y = standard_ops.matmul(x, w) if bias_init is not None: b = _bias_variable(shape=[num_output_units], dtype=dtype, initializer=bias_init, collections=bias_collections, regularizer=bias_regularizer) y = nn.bias_add(y, b) return _apply_activation(y, activation_fn, output_collections)
def convolution2d(x, num_output_channels, kernel_size, activation_fn=None, stride=(1, 1), padding='SAME', weight_init=initializers.xavier_initializer_conv2d(), bias_init=standard_ops.constant_initializer(0.), name=None, weight_collections=None, bias_collections=None, output_collections=None, weight_regularizer=None, bias_regularizer=None): """Adds the parameters for a conv2d layer and returns the output. A neural network convolution layer is generally defined as: \\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`, **conv2d** is `tf.nn.conv2d` and `x` has shape `[batch, height, width, channels]`. The output of this op is of shape `[batch, out_height, out_width, num_output_channels]`, where `out_width` and `out_height` are determined by the `padding` argument. See `conv2D` for details. This op creates `w` and optionally `b` and adds various summaries that can be useful for visualizing learning or diagnosing training problems. Bias can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which collections to place the created variables in (`weight_collections` and `bias_collections`). A per layer regularization can be specified by setting `weight_regularizer`. This is only applied to weights and not the bias. Args: x: A 4-D input `Tensor`. num_output_channels: The number of output channels (i.e. the size of the last dimension of the output). kernel_size: A length 2 `list` or `tuple` containing the kernel size. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. stride: A length 2 `list` or `tuple` specifying the stride of the sliding window across the image. padding: A `string` from: "SAME", "VALID". The type of padding algorithm to use. weight_init: An optional initialization. If not specified, uses Xavier initialization (see `tf.learn.xavier_initializer`). bias_init: An initializer for the bias, defaults to 0. Set to`None` in order to disable bias. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "convolution2d" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections to which weights are added. bias_collections: List of graph collections to which biases are added. output_collections: List of graph collections to which outputs are added. weight_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for weights. bias_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for biases. Returns: The result of applying a 2-D convolutional layer. Raises: ValueError: If `kernel_size` or `stride` are not length 2. """ with variable_scope.variable_op_scope([x], name, 'convolution2d'): num_input_channels = x.get_shape().dims[3].value if len(kernel_size) != 2: raise ValueError('kernel_size must be length 2: ' % kernel_size) if len(stride) != 2: raise ValueError('stride must be length 2: ' % kernel_size) stride = [1, stride[0], stride[1], 1] shape = [kernel_size[0], kernel_size[1], num_input_channels, num_output_channels] dtype = x.dtype.base_dtype w = _weight_variable(shape=shape, dtype=dtype, initializer=weight_init, collections=weight_collections, regularizer=weight_regularizer) y = nn.conv2d(x, w, stride, padding) if bias_init is not None: b = _bias_variable(shape=[num_output_channels], dtype=dtype, initializer=bias_init, collections=bias_collections, regularizer=bias_regularizer) y = nn.bias_add(y, b) return _apply_activation(y, activation_fn, output_collections)
def fully_connected(x, num_output_units, activation_fn=None, weight_init=initializers.xavier_initializer(), bias_init=standard_ops.constant_initializer(0.), name=None, weight_collections=(ops.GraphKeys.WEIGHTS,), bias_collections=(ops.GraphKeys.BIASES,), output_collections=(ops.GraphKeys.ACTIVATIONS,), weight_regularizer=None, bias_regularizer=None): # pylint: disable=anomalous-backslash-in-string r"""Adds the parameters for a fully connected layer and returns the output. A fully connected layer is generally defined as a matrix multiply: `y = f(w * x + b)` where `f` is given by `activation_fn`. If `activation_fn` is `None`, the result of `y = w * x + b` is returned. If `x` has shape [\\\(\\text{dim}_0, \\text{dim}_1, ..., \\text{dim}_n\\\)] with more than 2 dimensions (\\\(n > 1\\\)), then we repeat the matrix multiply along the first dimensions. The result r is a tensor of shape [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`], where \\\( r_{i_0, ..., i_{n-1}, k} = \\sum_{0 \\leq j < \\text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\\). This is accomplished by reshaping `x` to 2-D [\\\(\\text{dim}_0 \\cdot ... \\cdot \\text{dim}_{n-1}, \\text{dim}_n\\\)] before the matrix multiply and afterwards reshaping it to [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`]. This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which in collections to place the created variables (`weight_collections` and `bias_collections`; note that the variables are always added to the `VARIABLES` collection). The output of the layer can be placed in custom collections using `output_collections`. The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`, respectively. A per layer regularization can be specified by setting `weight_regularizer` and `bias_regularizer`, which are applied to the weights and biases respectively, and whose output is added to the `REGULARIZATION_LOSSES` collection. Args: x: The input `Tensor`. num_output_units: The size of the output. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. If None is used, do not apply any activation. weight_init: An optional weight initialization, defaults to `xavier_initializer`. bias_init: An initializer for the bias, defaults to 0. Set to `None` in order to disable bias. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "fully_connected" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections to which weights are added. bias_collections: List of graph collections to which biases are added. output_collections: List of graph collections to which outputs are added. weight_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for weights. bias_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for biases. Returns: The output of the fully connected layer. Raises: ValueError: if x has rank less than 2 or if its last dimension is not set. """ # pylint: enable=anomalous-backslash-in-string with variable_scope.variable_op_scope([x], name, 'fully_connected'): dims = x.get_shape().dims if dims is None: raise ValueError('dims of x must be known but is None') if len(dims) < 2: raise ValueError('rank of x must be at least 2 not: %d' % len(dims)) num_input_units = dims[-1].value if num_input_units is None: raise ValueError('last dimension of x must be known but is None') dtype = x.dtype.base_dtype weight_collections = set(list(weight_collections or []) + [ops.GraphKeys.VARIABLES]) w = variable_scope.get_variable('weights', shape=[num_input_units, num_output_units], dtype=dtype, initializer=weight_init, collections=weight_collections, regularizer=weight_regularizer) x_2_dim = x if len(dims) <= 2 else array_ops.reshape(x, [-1, num_input_units]) y = standard_ops.matmul(x_2_dim, w) if bias_init is not None: bias_collections = set(list(bias_collections or []) + [ops.GraphKeys.VARIABLES]) b = variable_scope.get_variable('bias', shape=[num_output_units], dtype=dtype, initializer=bias_init, collections=bias_collections, regularizer=bias_regularizer) y = nn.bias_add(y, b) if len(dims) > 2: out_shape = array_ops.unpack(array_ops.shape(x)) out_shape[-1] = num_output_units y = array_ops.reshape(y, array_ops.pack(out_shape)) static_shape = x.get_shape().as_list() static_shape[-1] = num_output_units y.set_shape(static_shape) return _apply_activation(y, activation_fn, output_collections)
def fully_connected(x, num_output_units, activation_fn=None, weight_init=initializers.xavier_initializer(), bias_init=standard_ops.constant_initializer(0.), name=None, weight_collections=(ops.GraphKeys.WEIGHTS,), bias_collections=(ops.GraphKeys.BIASES,), output_collections=(ops.GraphKeys.ACTIVATIONS,), trainable=True, weight_regularizer=None, bias_regularizer=None): # pylint: disable=anomalous-backslash-in-string r"""Adds the parameters for a fully connected layer and returns the output. A fully connected layer is generally defined as a matrix multiply: `y = f(w * x + b)` where `f` is given by `activation_fn`. If `activation_fn` is `None`, the result of `y = w * x + b` is returned. If `x` has shape [\\\(\\text{dim}_0, \\text{dim}_1, ..., \\text{dim}_n\\\)] with more than 2 dimensions (\\\(n > 1\\\)), then we repeat the matrix multiply along the first dimensions. The result r is a tensor of shape [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`], where \\\( r_{i_0, ..., i_{n-1}, k} = \\sum_{0 \\leq j < \\text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\\). This is accomplished by reshaping `x` to 2-D [\\\(\\text{dim}_0 \\cdot ... \\cdot \\text{dim}_{n-1}, \\text{dim}_n\\\)] before the matrix multiply and afterwards reshaping it to [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`]. This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting `bias_init` to `None`. The variable creation is compatible with `tf.variable_scope` and so can be reused with `tf.variable_scope` or `tf.make_template`. Most of the details of variable creation can be controlled by specifying the initializers (`weight_init` and `bias_init`) and which in collections to place the created variables (`weight_collections` and `bias_collections`; note that the variables are always added to the `VARIABLES` collection). The output of the layer can be placed in custom collections using `output_collections`. The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`, respectively. A per layer regularization can be specified by setting `weight_regularizer` and `bias_regularizer`, which are applied to the weights and biases respectively, and whose output is added to the `REGULARIZATION_LOSSES` collection. Args: x: The input `Tensor`. num_output_units: The size of the output. activation_fn: A function that requires a single Tensor that is applied as a non-linearity. If None is used, do not apply any activation. weight_init: An optional weight initialization, defaults to `xavier_initializer`. bias_init: An initializer for the bias, defaults to 0. Set to `None` in order to disable bias. name: The name for this operation is used to name operations and to find variables. If specified it must be unique for this scope, otherwise a unique name starting with "fully_connected" will be created. See `tf.variable_op_scope` for details. weight_collections: List of graph collections to which weights are added. bias_collections: List of graph collections to which biases are added. output_collections: List of graph collections to which outputs are added. trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). weight_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for weights. bias_regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. Used for biases. Returns: The output of the fully connected layer. Raises: ValueError: if x has rank less than 2 or if its last dimension is not set. """ # pylint: enable=anomalous-backslash-in-string with variable_scope.variable_op_scope([x], name, 'fully_connected'): dims = x.get_shape().dims if dims is None: raise ValueError('dims of x must be known but is None') if len(dims) < 2: raise ValueError('rank of x must be at least 2 not: %d' % len(dims)) num_input_units = dims[-1].value if num_input_units is None: raise ValueError('last dimension of x must be known but is None') dtype = x.dtype.base_dtype weight_collections = set(list(weight_collections or []) + [ops.GraphKeys.VARIABLES]) w = variable_scope.get_variable('weights', shape=[num_input_units, num_output_units], dtype=dtype, initializer=weight_init, collections=weight_collections, regularizer=weight_regularizer, trainable=trainable) x_2_dim = x if len(dims) <= 2 else array_ops.reshape(x, [-1, num_input_units]) y = standard_ops.matmul(x_2_dim, w) if bias_init is not None: bias_collections = set(list(bias_collections or []) + [ops.GraphKeys.VARIABLES]) b = variable_scope.get_variable('bias', shape=[num_output_units], dtype=dtype, initializer=bias_init, collections=bias_collections, regularizer=bias_regularizer, trainable=trainable) y = nn.bias_add(y, b) if len(dims) > 2: out_shape = array_ops.unpack(array_ops.shape(x)) out_shape[-1] = num_output_units y = array_ops.reshape(y, array_ops.pack(out_shape)) static_shape = x.get_shape().as_list() static_shape[-1] = num_output_units y.set_shape(static_shape) return _apply_activation(y, activation_fn, output_collections)