def __init__( self, incoming, axes='auto', epsilon=1e-4, alpha=0.1, beta=init.Constant(0), gamma=init.Constant(1), mean=init.Constant(0), inv_std=init.Constant(1), mode='high_mem', # [DV] add mode support **kwargs): self.mode = mode super(BatchNormLayer_Theano, self).__init__(incoming, **kwargs) if axes == 'auto': # default: normalize over all but the second axis axes = (0, ) + tuple(range(2, len(self.input_shape))) elif isinstance(axes, int): axes = (axes, ) self.axes = axes self.epsilon = epsilon self.alpha = alpha # create parameters, ignoring all dimensions in axes shape = [ size for axis, size in enumerate(self.input_shape) if axis not in self.axes ] # remove all dimensions in axes if any(size is None for size in shape): raise ValueError("BatchNormLayer needs specified input sizes for " "all axes not normalized over.") if beta is None: self.beta = None else: self.beta = self.add_param(beta, shape, 'beta', trainable=True, regularizable=False) if gamma is None: self.gamma = None else: self.gamma = self.add_param(gamma, shape, 'gamma', trainable=True, regularizable=True) self.mean = self.add_param(mean, shape, 'mean', trainable=False, regularizable=False) self.inv_std = self.add_param(inv_std, shape, 'inv_std', trainable=False, regularizable=False)
def __init__(self, incoming, num_filters, filter_size, stride=(1, 1), crop=0, untie_biases=False, W=init.GlorotUniform(), b=init.Constant(0.), nonlinearity=nonlinearities.rectify, flip_filters=False, output_size=None, **kwargs): # output_size must be set before calling the super constructor if (not isinstance(output_size, T.Variable) and output_size is not None): output_size = as_tuple(output_size, 2, int) self.output_size = output_size super(TransposedConv2DLayer, self).__init__(incoming, num_filters, filter_size, stride, crop, untie_biases, W, b, nonlinearity, flip_filters, n=2, **kwargs) # rename self.pad to self.crop: self.crop = self.pad del self.pad
def __init__(self, incoming, num_filters, filter_size, stride=(1, 1), pad=0, untie_biases=False, W=init.GlorotUniform(), b=init.Constant(0.), nonlinearity=nonlinearities.rectify, flip_filters=True, convolution=T.nnet.conv2d, **kwargs): super(Conv2DLayer, self).__init__(incoming, num_filters, filter_size, stride, pad, untie_biases, W, b, nonlinearity, flip_filters, n=2, **kwargs) self.convolution = convolution
def __init__(self, name, input_dims, output_dims, config=None, w=None, w_init=None, w_regularizer=None, b=None, b_init=None, b_regularizer=None, use_bias=True): super().__init__(name) # super(name) self.input_dims = input_dims self.output_dims = output_dims self.use_bias = use_bias if w_init is None: w_init = init.Gaussian(fan_in=input_dims) if b_init is None: b_init = init.Constant(0.0) self.param('w', (input_dims, output_dims), init_f=w_init, value=w) self.regularize(self._w, w_regularizer) if use_bias: self.param('b', (output_dims, ), init_f=b_init, value=b) self.regularize(self._b, b_regularizer) self.config_w = config.copy() self.config_b = config.copy()
def __init__(self, incoming, num_filters, filter_size, stride=1, pad=0, untie_biases=False, W=init.GlorotUniform(), b=init.Constant(0.), nonlinearity=nonlinearities.rectify, flip_filters=True, n=None, **kwargs): super(BaseConvLayer, self).__init__(incoming, **kwargs) if nonlinearity is None: self.nonlinearity = nonlinearities.identity else: self.nonlinearity = nonlinearity if n is None: n = len(self.input_shape) - 2 elif n != len(self.input_shape) - 2: raise ValueError("Tried to create a %dD convolution layer with " "input shape %r. Expected %d input dimensions " "(batchsize, channels, %d spatial dimensions)." % (n, self.input_shape, n + 2, n)) self.n = n self.num_filters = num_filters self.filter_size = as_tuple(filter_size, n, int) self.flip_filters = flip_filters self.stride = as_tuple(stride, n, int) self.untie_biases = untie_biases if pad == 'same': if any(s % 2 == 0 for s in self.filter_size): raise NotImplementedError( '`same` padding requires odd filter size.') if pad == 'valid': self.pad = as_tuple(0, n) elif pad in ('full', 'same'): self.pad = pad else: self.pad = as_tuple(pad, n, int) self.W = self.add_param(W, self.get_W_shape(), name="W") if b is None: self.b = None else: if self.untie_biases: biases_shape = (num_filters, ) + self.output_shape[2:] else: biases_shape = (num_filters, ) self.b = self.add_param(b, biases_shape, name="b", regularizable=False)
def __init__(self, input_layer, num_units, W=init.Normal(0.01), b=init.Constant(0.), nonlinearity=nonlinearities.rectify): super(DenseLayer, self).__init__(input_layer) if nonlinearity is None: self.nonlinearity = nonlinearities.identity else: self.nonlinearity = nonlinearity self.num_units = num_units num_inputs = self.input_layer.get_output_shape()[1] self.W = self.create_param(W, (num_inputs, num_units)) self.b = self.create_param(b, (num_units,))
def __init__(self, incoming, num_units, W=init.GlorotUniform(), b=init.Constant(0.), nonlinearity=nonlinearities.rectify, num_leading_axes=1, **kwargs): super(DenseLayer, self).__init__(incoming, **kwargs) self.nonlinearity = (nonlinearities.identity if nonlinearity is None else nonlinearity) self.num_units = num_units if num_leading_axes >= len(self.input_shape): raise ValueError( "Got num_leading_axes=%d for a %d-dimensional input, " "leaving no trailing axes for the dot product." % (num_leading_axes, len(self.input_shape))) elif num_leading_axes < -len(self.input_shape): raise ValueError( "Got num_leading_axes=%d for a %d-dimensional input, " "requesting more trailing axes than there are input " "dimensions." % (num_leading_axes, len(self.input_shape))) self.num_leading_axes = num_leading_axes if any(s is None for s in self.input_shape[num_leading_axes:]): raise ValueError( "A DenseLayer requires a fixed input shape (except for " "the leading axes). Got %r for num_leading_axes=%d." % (self.input_shape, self.num_leading_axes)) num_inputs = int(np.prod(self.input_shape[num_leading_axes:])) self.W = self.add_param(W, (num_inputs, num_units), name="W") if b is None: self.b = None else: self.b = self.add_param(b, (num_units, ), name="b", regularizable=False)
def __init__(self, incoming, num_filters, filter_size, dilation=(1, 1), pad=0, untie_biases=False, W=init.GlorotUniform(), b=init.Constant(0.), nonlinearity=nonlinearities.rectify, flip_filters=False, **kwargs): self.dilation = as_tuple(dilation, 2, int) super(DilatedConv2DLayer, self).__init__(incoming, num_filters, filter_size, 1, pad, untie_biases, W, b, nonlinearity, flip_filters, n=2, **kwargs) # remove self.stride: del self.stride # require valid convolution if self.pad != (0, 0): raise NotImplementedError( "DilatedConv2DLayer requires pad=0 / (0,0) / 'valid', but " "got %r. For a padded dilated convolution, add a PadLayer." % (pad, )) # require unflipped filters if self.flip_filters: raise NotImplementedError( "DilatedConv2DLayer requires flip_filters=False.")
def constant_param(value=0.0, shape=(0, )): return theano.shared(init.Constant(value).sample(shape), borrow=True)