def __init__(
self,
incoming,
axes='auto',
epsilon=1e-4,
alpha=0.1,
beta=init.Constant(0),
gamma=init.Constant(1),
mean=init.Constant(0),
inv_std=init.Constant(1),
mode='high_mem', # [DV] add mode support
**kwargs):
self.mode = mode
super(BatchNormLayer_Theano, self).__init__(incoming, **kwargs)
if axes == 'auto':
# default: normalize over all but the second axis
axes = (0, ) + tuple(range(2, len(self.input_shape)))
elif isinstance(axes, int):
axes = (axes, )
self.axes = axes
self.epsilon = epsilon
self.alpha = alpha
# create parameters, ignoring all dimensions in axes
shape = [
size for axis, size in enumerate(self.input_shape)
if axis not in self.axes
] # remove all dimensions in axes
if any(size is None for size in shape):
raise ValueError("BatchNormLayer needs specified input sizes for "
"all axes not normalized over.")
if beta is None:
self.beta = None
else:
self.beta = self.add_param(beta,
shape,
'beta',
trainable=True,
regularizable=False)
if gamma is None:
self.gamma = None
else:
self.gamma = self.add_param(gamma,
shape,
'gamma',
trainable=True,
regularizable=True)
self.mean = self.add_param(mean,
shape,
'mean',
trainable=False,
regularizable=False)
self.inv_std = self.add_param(inv_std,
shape,
'inv_std',
trainable=False,
regularizable=False)
def __init__(self,
incoming,
num_filters,
filter_size,
stride=(1, 1),
crop=0,
untie_biases=False,
W=init.GlorotUniform(),
b=init.Constant(0.),
nonlinearity=nonlinearities.rectify,
flip_filters=False,
output_size=None,
**kwargs):
# output_size must be set before calling the super constructor
if (not isinstance(output_size, T.Variable)
and output_size is not None):
output_size = as_tuple(output_size, 2, int)
self.output_size = output_size
super(TransposedConv2DLayer, self).__init__(incoming,
num_filters,
filter_size,
stride,
crop,
untie_biases,
W,
b,
nonlinearity,
flip_filters,
n=2,
**kwargs)
# rename self.pad to self.crop:
self.crop = self.pad
del self.pad
def __init__(self,
incoming,
num_filters,
filter_size,
stride=(1, 1),
pad=0,
untie_biases=False,
W=init.GlorotUniform(),
b=init.Constant(0.),
nonlinearity=nonlinearities.rectify,
flip_filters=True,
convolution=T.nnet.conv2d,
**kwargs):
super(Conv2DLayer, self).__init__(incoming,
num_filters,
filter_size,
stride,
pad,
untie_biases,
W,
b,
nonlinearity,
flip_filters,
n=2,
**kwargs)
self.convolution = convolution
def __init__(self,
name,
input_dims,
output_dims,
config=None,
w=None,
w_init=None,
w_regularizer=None,
b=None,
b_init=None,
b_regularizer=None,
use_bias=True):
super().__init__(name)
# super(name)
self.input_dims = input_dims
self.output_dims = output_dims
self.use_bias = use_bias
if w_init is None: w_init = init.Gaussian(fan_in=input_dims)
if b_init is None: b_init = init.Constant(0.0)
self.param('w', (input_dims, output_dims), init_f=w_init, value=w)
self.regularize(self._w, w_regularizer)
if use_bias:
self.param('b', (output_dims, ), init_f=b_init, value=b)
self.regularize(self._b, b_regularizer)
self.config_w = config.copy()
self.config_b = config.copy()
def __init__(self,
incoming,
num_filters,
filter_size,
stride=1,
pad=0,
untie_biases=False,
W=init.GlorotUniform(),
b=init.Constant(0.),
nonlinearity=nonlinearities.rectify,
flip_filters=True,
n=None,
**kwargs):
super(BaseConvLayer, self).__init__(incoming, **kwargs)
if nonlinearity is None:
self.nonlinearity = nonlinearities.identity
else:
self.nonlinearity = nonlinearity
if n is None:
n = len(self.input_shape) - 2
elif n != len(self.input_shape) - 2:
raise ValueError("Tried to create a %dD convolution layer with "
"input shape %r. Expected %d input dimensions "
"(batchsize, channels, %d spatial dimensions)." %
(n, self.input_shape, n + 2, n))
self.n = n
self.num_filters = num_filters
self.filter_size = as_tuple(filter_size, n, int)
self.flip_filters = flip_filters
self.stride = as_tuple(stride, n, int)
self.untie_biases = untie_biases
if pad == 'same':
if any(s % 2 == 0 for s in self.filter_size):
raise NotImplementedError(
'`same` padding requires odd filter size.')
if pad == 'valid':
self.pad = as_tuple(0, n)
elif pad in ('full', 'same'):
self.pad = pad
else:
self.pad = as_tuple(pad, n, int)
self.W = self.add_param(W, self.get_W_shape(), name="W")
if b is None:
self.b = None
else:
if self.untie_biases:
biases_shape = (num_filters, ) + self.output_shape[2:]
else:
biases_shape = (num_filters, )
self.b = self.add_param(b,
biases_shape,
name="b",
regularizable=False)
def __init__(self, input_layer, num_units, W=init.Normal(0.01), b=init.Constant(0.), nonlinearity=nonlinearities.rectify):
super(DenseLayer, self).__init__(input_layer)
if nonlinearity is None:
self.nonlinearity = nonlinearities.identity
else:
self.nonlinearity = nonlinearity
self.num_units = num_units
num_inputs = self.input_layer.get_output_shape()[1]
self.W = self.create_param(W, (num_inputs, num_units))
self.b = self.create_param(b, (num_units,))
def __init__(self,
incoming,
num_units,
W=init.GlorotUniform(),
b=init.Constant(0.),
nonlinearity=nonlinearities.rectify,
num_leading_axes=1,
**kwargs):
super(DenseLayer, self).__init__(incoming, **kwargs)
self.nonlinearity = (nonlinearities.identity
if nonlinearity is None else nonlinearity)
self.num_units = num_units
if num_leading_axes >= len(self.input_shape):
raise ValueError(
"Got num_leading_axes=%d for a %d-dimensional input, "
"leaving no trailing axes for the dot product." %
(num_leading_axes, len(self.input_shape)))
elif num_leading_axes < -len(self.input_shape):
raise ValueError(
"Got num_leading_axes=%d for a %d-dimensional input, "
"requesting more trailing axes than there are input "
"dimensions." % (num_leading_axes, len(self.input_shape)))
self.num_leading_axes = num_leading_axes
if any(s is None for s in self.input_shape[num_leading_axes:]):
raise ValueError(
"A DenseLayer requires a fixed input shape (except for "
"the leading axes). Got %r for num_leading_axes=%d." %
(self.input_shape, self.num_leading_axes))
num_inputs = int(np.prod(self.input_shape[num_leading_axes:]))
self.W = self.add_param(W, (num_inputs, num_units), name="W")
if b is None:
self.b = None
else:
self.b = self.add_param(b, (num_units, ),
name="b",
regularizable=False)
def __init__(self,
incoming,
num_filters,
filter_size,
dilation=(1, 1),
pad=0,
untie_biases=False,
W=init.GlorotUniform(),
b=init.Constant(0.),
nonlinearity=nonlinearities.rectify,
flip_filters=False,
**kwargs):
self.dilation = as_tuple(dilation, 2, int)
super(DilatedConv2DLayer, self).__init__(incoming,
num_filters,
filter_size,
1,
pad,
untie_biases,
W,
b,
nonlinearity,
flip_filters,
n=2,
**kwargs)
# remove self.stride:
del self.stride
# require valid convolution
if self.pad != (0, 0):
raise NotImplementedError(
"DilatedConv2DLayer requires pad=0 / (0,0) / 'valid', but "
"got %r. For a padded dilated convolution, add a PadLayer." %
(pad, ))
# require unflipped filters
if self.flip_filters:
raise NotImplementedError(
"DilatedConv2DLayer requires flip_filters=False.")
def constant_param(value=0.0, shape=(0, )):
return theano.shared(init.Constant(value).sample(shape), borrow=True)