def __call__(self, input_layer, init=None, stddev=None, l2loss=None): """Performs a diagonal matrix multiplication with a learned vector. This creates the parameter vector. Args: input_layer: The input_layer. init: An optional initialization. If not specified, uses Xavier initialization. stddev: A standard deviation to use in parameter initialization. l2loss: An l2 weight decay to apply. Returns: A Pretty Tensor handle to the layer. Raises: ValueError: if the head_shape is not rank 2 or the number of input nodes (second dim) is not known. """ size = input_layer.shape[-1] if init is None: if stddev is None: init = layers.xavier_init(size, 0) elif stddev: init = tf.truncated_normal_initializer(stddev=stddev) else: init = tf.zeros_initializer param = self.variable('weights', [size], init) layers.add_l2loss(input_layer.bookkeeper, param, l2loss) return input_layer.with_tensor(input_layer * param, parameters=self.vars)
def __call__(self, input_layer, weights=None, l2loss=None, phase=prettytensor.Phase.train, parameter_modifier=parameters.identity): """Performs a diagonal matrix multiplication with a learned vector. This creates the parameter vector. Args: input_layer: The input_layer. weights: An initializer for weights or a Tensor. If not specified, uses Xavier initialization. l2loss: An l2 weight decay to apply. phase: The phase of graph construction. See `pt.Phase`. parameter_modifier: A function to modify parameters that is applied after creation and before use. Returns: A Pretty Tensor handle to the layer. Raises: ValueError: if this is not rank 2 or the number of input nodes (second dim) is not known. """ size = input_layer.shape[-1] if weights is None: weights = layers.xavier_init(size, 0) param = parameter_modifier('weights', self.variable('weights', [size], weights), phase) layers.add_l2loss(input_layer.bookkeeper, param, l2loss) return input_layer.with_tensor(input_layer * param, parameters=self.vars)
def __call__( self, input_layer, weights=None, l2loss=None, phase=prettytensor.Phase.train, parameter_modifier=parameters.identity, ): """Performs a diagonal matrix multiplication with a learned vector. This creates the parameter vector. Args: input_layer: The input_layer. weights: An initializer for weights or a Tensor. If not specified, uses Xavier initialization. l2loss: An l2 weight decay to apply. phase: The phase of graph construction. See `pt.Phase`. parameter_modifier: A function to modify parameters that is applied after creation and before use. Returns: A Pretty Tensor handle to the layer. Raises: ValueError: if this is not rank 2 or the number of input nodes (second dim) is not known. """ size = input_layer.shape[-1] if weights is None: weights = layers.xavier_init(size, 0) param = parameter_modifier("weights", self.variable("weights", [size], weights), phase) layers.add_l2loss(input_layer.bookkeeper, param, l2loss) return input_layer.with_tensor(input_layer * param, parameters=self.vars)
def __call__(self, input_layer, kernel, depth, name=PROVIDED, stride=None, activation_fn=None, l2loss=None, init=None, stddev=None, bias=True, edges=PAD_SAME, batch_normalize=False, phase=Phase.train): """Adds a convolution to the stack of operations. The current head must be a rank 4 Tensor. Args: input_layer: The chainable object, supplied. kernel: The size of the patch for the pool, either an int or a length 1 or 2 sequence (if length 1 or int, it is expanded). depth: The depth of the new Tensor. name: The name for this operation is also used to create/find the parameter variables. stride: The strides as a length 1, 2 or 4 sequence or an integer. If an int, length 1 or 2, the stride in the first and last dimensions are 1. activation_fn: A tuple of (activation_function, extra_parameters). Any function that takes a tensor as its first argument can be used. More common functions will have summaries added (e.g. relu). l2loss: Set to a value greater than 0 to use L2 regularization to decay the weights. init: An optional initialization. If not specified, uses Xavier initialization. stddev: A standard deviation to use in parameter initialization. bias: Set to False to not have a bias. edges: Either SAME to use 0s for the out of bounds area or VALID to shrink the output size and only uses valid input pixels. batch_normalize: Set to True to batch_normalize this layer. Returns: Handle to the generated layer. Raises: ValueError: If head is not a rank 4 tensor or the depth of the input (4th dim) is not known. """ if len(input_layer.shape) != 4: raise ValueError('Cannot perform conv2d on tensor with shape %s' % input_layer.shape) if input_layer.shape[3] is None: raise ValueError('Input depth must be known') kernel = _kernel(kernel) stride = _stride(stride) size = [kernel[0], kernel[1], depth, input_layer.shape[3]] books = input_layer.bookkeeper if init is None: if stddev is None: patch_size = size[0] * size[1] init = layers.xavier_init(size[2] * patch_size, size[3] * patch_size) elif stddev: init = tf.truncated_normal_initializer(stddev=stddev) else: init = tf.zeros_initializer elif stddev is not None: raise ValueError('Do not set both init and stddev.') dtype = input_layer.tensor.dtype params = self.variable('weights', size, init, dt=dtype) input_height = input_layer.shape[1] input_width = input_layer.shape[2] filter_height = kernel[0] filter_width = kernel[1] row_stride = stride[1] col_stride = stride[2] out_rows, out_cols = get2d_deconv_output_size(input_height, input_width, filter_height, filter_width, row_stride, col_stride, edges) output_shape = [input_layer.shape[0], out_rows, out_cols, depth] y = tf.nn.conv2d_transpose(input_layer, params, output_shape, stride, edges) layers.add_l2loss(books, params, l2loss) if bias: y += self.variable('bias', [size[-2]], tf.zeros_initializer, dt=dtype) books.add_scalar_summary(tf.reduce_mean(layers.spatial_slice_zeros(y)), '%s/zeros_spatial' % y.op.name) if batch_normalize: y = input_layer.with_tensor(y).batch_normalize(phase=phase) if activation_fn is not None: if not isinstance(activation_fn, collections.Sequence): activation_fn = (activation_fn, ) y = layers.apply_activation(books, y, activation_fn[0], activation_args=activation_fn[1:]) return input_layer.with_tensor(y)
def __call__(self, input_layer, size, activation_fn=None, l2loss=None, init=None, stddev=None, bias=True, bias_init=0., transpose_weights=False, name=PROVIDED): """Adds the parameters for a fully connected layer and returns a tensor. The current head must be a rank 2 Tensor. Args: input_layer: The Pretty Tensor object, supplied. size: The number of neurons activation_fn: A tuple of (activation_function, extra_parameters). Any function that takes a tensor as its first argument can be used. More common functions will have summaries added (e.g. relu). l2loss: Set to a value greater than 0 to use L2 regularization to decay the weights. init: An optional initialization. If not specified, uses Xavier initialization. stddev: A standard deviation to use in parameter initialization. bias: Set to False to not have a bias. bias_init: The initial value for the bias. transpose_weights: Flag indicating if weights should be transposed; this is useful for loading models with a different shape. name: The name for this operation is also used to create/find the parameter variables. Returns: A Pretty Tensor handle to the layer. Raises: ValueError: if the head_shape is not rank 2 or the number of input nodes (second dim) is not known. """ if input_layer.get_shape().ndims != 2: raise ValueError( 'fully_connected requires a rank 2 Tensor with known second ' 'dimension: %s' % input_layer.get_shape()) in_size = input_layer.shape[1] if input_layer.shape[1] is None: raise ValueError('Number of input nodes must be known.') books = input_layer.bookkeeper if init is None: if stddev is None: init = layers.xavier_init(in_size, size) elif stddev: init = tf.truncated_normal_initializer(stddev=stddev) else: init = tf.zeros_initializer elif stddev is not None: raise ValueError('Do not set both init and stddev.') dtype = input_layer.tensor.dtype weight_shape = [size, in_size] if transpose_weights else [in_size, size] params = self.variable( 'weights', weight_shape, init, dt=dtype) y = tf.matmul(input_layer, params, transpose_b=transpose_weights) layers.add_l2loss(books, params, l2loss) if bias: y += self.variable( 'bias', [size], tf.constant_initializer(bias_init), dt=dtype) if activation_fn is not None: if not isinstance(activation_fn, collections.Sequence): activation_fn = (activation_fn,) y = layers.apply_activation( books, y, activation_fn[0], activation_args=activation_fn[1:]) books.add_histogram_summary(y, '%s/activations' % y.op.name) return input_layer.with_tensor(y, parameters=self.vars)
def __call__(self, input_layer, kernel, depth, activation_fn=None, stride=None, l2loss=None, init=None, stddev=None, bias=True, edges=PAD_SAME, batch_normalize=False, name=PROVIDED): """Adds a convolution to the stack of operations. The current head must be a rank 4 Tensor. Args: input_layer: The chainable object, supplied. kernel: The size of the patch for the pool, either an int or a length 1 or 2 sequence (if length 1 or int, it is expanded). depth: The depth of the new Tensor. activation_fn: A tuple of (activation_function, extra_parameters). Any function that takes a tensor as its first argument can be used. More common functions will have summaries added (e.g. relu). stride: The strides as a length 1, 2 or 4 sequence or an integer. If an int, length 1 or 2, the stride in the first and last dimensions are 1. l2loss: Set to a value greater than 0 to use L2 regularization to decay the weights. init: An optional initialization. If not specified, uses Xavier initialization. stddev: A standard deviation to use in parameter initialization. bias: Set to False to not have a bias. edges: Either SAME to use 0s for the out of bounds area or VALID to shrink the output size and only uses valid input pixels. batch_normalize: Set to True to batch_normalize this layer. name: The name for this operation is also used to create/find the parameter variables. Returns: Handle to the generated layer. Raises: ValueError: If head is not a rank 4 tensor or the depth of the input (4th dim) is not known. """ if len(input_layer.shape) != 4: raise ValueError( 'Cannot perform conv2d on tensor with shape %s' % input_layer.shape) if input_layer.shape[3] is None: raise ValueError('Input depth must be known') kernel = _kernel(kernel) stride = _stride(stride) size = [kernel[0], kernel[1], input_layer.shape[3], depth] books = input_layer.bookkeeper if init is None: if stddev is None: patch_size = size[0] * size[1] init = layers.xavier_init(size[2] * patch_size, size[3] * patch_size) elif stddev: init = tf.truncated_normal_initializer(stddev=stddev) else: init = tf.zeros_initializer elif stddev is not None: raise ValueError('Do not set both init and stddev.') dtype = input_layer.tensor.dtype params = self.variable('weights', size, init, dt=dtype) y = tf.nn.conv2d(input_layer, params, stride, edges) layers.add_l2loss(books, params, l2loss) if bias: y += self.variable( 'bias', [size[-1]], tf.zeros_initializer, dt=dtype) books.add_scalar_summary( tf.reduce_mean( layers.spatial_slice_zeros(y)), '%s/zeros_spatial' % y.op.name) if batch_normalize: y = input_layer.with_tensor(y).batch_normalize() if activation_fn is not None: if not isinstance(activation_fn, collections.Sequence): activation_fn = (activation_fn,) y = layers.apply_activation( books, y, activation_fn[0], activation_args=activation_fn[1:]) books.add_histogram_summary(y, '%s/activations' % y.op.name) return input_layer.with_tensor(y, parameters=self.vars)
def __call__(self, input_layer, size, name=PROVIDED, activation_fn=None, l2loss=None, init=None, stddev=None, bias=True, bias_init=0.): """Adds the parameters for a fully connected layer and returns a tensor. The current head must be a rank 2 Tensor. Args: input_layer: The Pretty Tensor object, supplied. size: The number of neurons name: The name for this operation is also used to create/find the parameter variables. activation_fn: A tuple of (activation_function, extra_parameters). Any function that takes a tensor as its first argument can be used. More common functions will have summaries added (e.g. relu). l2loss: Set to a value greater than 0 to use L2 regularization to decay the weights. init: An optional initialization. If not specified, uses Xavier initialization. stddev: A standard deviation to use in parameter initialization. bias: Set to False to not have a bias. bias_init: The initial value for the bias. Returns: A Pretty Tensor handle to the layer. Raises: ValueError: if the head_shape is not rank 2 or the number of input nodes (second dim) is not known. """ if len(input_layer.shape) != 2: raise ValueError( 'Cannot perform fully connected on tensor with shape %s' % input_layer.shape) in_size = input_layer.shape[1] if input_layer.shape[1] is None: raise ValueError('Number of input nodes must be known.') books = input_layer.bookkeeper if init is None: if stddev is None: init = layers.xavier_init(in_size, size) elif stddev: init = tf.truncated_normal_initializer(stddev=stddev) else: init = tf.zeros_initializer elif stddev is not None: raise ValueError('Do not set both init and stddev.') dtype = input_layer.tensor.dtype params = self.variable( 'weights', [in_size, size], init, dt=dtype) y = tf.matmul(input_layer, params) layers.add_l2loss(books, params, l2loss) if bias: y += self.variable( 'bias', [size], tf.constant_initializer(bias_init), dt=dtype) if activation_fn is not None: if not isinstance(activation_fn, collections.Sequence): activation_fn = (activation_fn,) return layers.apply_activation( books, y, activation_fn[0], activation_args=activation_fn[1:]) else: return y
def __call__(self, input_layer, size, activation_fn=None, l2loss=None, init=None, stddev=None, bias=True, bias_init=0., name=PROVIDED): """Adds the parameters for a fully connected layer and returns a tensor. The current head must be a rank 2 Tensor. Args: input_layer: The Pretty Tensor object, supplied. size: The number of neurons activation_fn: A tuple of (activation_function, extra_parameters). Any function that takes a tensor as its first argument can be used. More common functions will have summaries added (e.g. relu). l2loss: Set to a value greater than 0 to use L2 regularization to decay the weights. init: An optional initialization. If not specified, uses Xavier initialization. stddev: A standard deviation to use in parameter initialization. bias: Set to False to not have a bias. bias_init: The initial value for the bias. name: The name for this operation is also used to create/find the parameter variables. Returns: A Pretty Tensor handle to the layer. Raises: ValueError: if the head_shape is not rank 2 or the number of input nodes (second dim) is not known. """ if len(input_layer.shape) != 2: raise ValueError( 'Cannot perform fully connected on tensor with shape %s' % input_layer.shape) in_size = input_layer.shape[1] if input_layer.shape[1] is None: raise ValueError('Number of input nodes must be known.') books = input_layer.bookkeeper if init is None: if stddev is None: init = layers.xavier_init(in_size, size) elif stddev: init = tf.truncated_normal_initializer(stddev=stddev) else: init = tf.zeros_initializer elif stddev is not None: raise ValueError('Do not set both init and stddev.') dtype = input_layer.tensor.dtype params = self.variable('weights', [in_size, size], init, dt=dtype) y = tf.matmul(input_layer, params) layers.add_l2loss(books, params, l2loss) if bias: y += self.variable('bias', [size], tf.constant_initializer(bias_init), dt=dtype) if activation_fn is not None: if not isinstance(activation_fn, collections.Sequence): activation_fn = (activation_fn, ) y = layers.apply_activation(books, y, activation_fn[0], activation_args=activation_fn[1:]) books.add_histogram_summary(y, '%s/activations' % y.op.name) return input_layer.with_tensor(y, parameters=self.vars)