def f_conv(self, x, spec, in_dim, weight_name):
layer_type, dims = spec
num_filters = dims[0]
filter_size = (dims[1], dims[1])
stride = (dims[2], dims[2])
bm = 'full' if 'convf' in layer_type else 'valid'
num_channels = in_dim[0]
W = self.weight(self.rand_init_conv(
(num_filters, num_channels) + filter_size), weight_name)
if stride != (1, 1):
f = GpuCorrMM(subsample=stride, border_mode=bm, pad=(0, 0))
y = f(gpu_contiguous(x), gpu_contiguous(W))
else:
assert self.p.batch_size == self.p.valid_batch_size
y = conv2d(x, W, image_shape=(2*self.p.batch_size, ) + in_dim,
filter_shape=((num_filters, num_channels) +
filter_size), border_mode=bm)
output_size = ((num_filters,) +
ConvOp.getOutputShape(in_dim[1:], filter_size,
stride, bm))
return y, output_size
def local_conv2d_gradinputs_cpu(node):
kern, topgrad, shape = node.inputs
if ((not isinstance(kern.type, TensorType)
or not isinstance(topgrad.type, TensorType))):
return None
if node.op.border_mode not in ['full', 'valid']:
return None
if not node.op.filter_flip:
# Not tested yet
return None
# Conv 3d implementation, needed when subsample > 2
if node.op.border_mode == 'valid' and node.op.subsample != (1, 1):
kern = kern[:, :, ::-1, ::-1]
shuffled_kern = kern.dimshuffle(0, 2, 3, 'x', 1)
shuffled_topgrad = topgrad.dimshuffle(0, 2, 3, 'x', 1)
b = theano.tensor.zeros_like(shuffled_kern[0, 0, 0, 0, :])
rval = convTransp3D(W=shuffled_kern,
b=b,
d=(node.op.subsample[0], node.op.subsample[1], 1),
H=shuffled_topgrad,
RShape=(shape[0], shape[1], 1))
rval = theano.tensor.addbroadcast(rval, 3)
rval = rval.dimshuffle(0, 4, 1, 2)
rval = patternbroadcast(rval, node.outputs[0].broadcastable)
return [rval]
# Conv2d Implementation
dx, dy = node.op.subsample
if dx not in (1, 2) or dy not in (1, 2):
# Not implemented in the gradient of ConvOp
return None
if node.op.imshp is None:
op_imshp = (None, None, None, None)
else:
op_imshp = node.op.imshp
if node.op.kshp is None:
op_kshp = (None, None, None, None)
else:
op_kshp = node.op.kshp
if None in op_imshp or None in op_kshp:
if (dx, dy) != (1, 1):
return None
mode = 'valid'
if not node.op.border_mode == 'full':
mode = 'full'
filters = kern.dimshuffle((1, 0, 2, 3))
filters = filters[:, :, ::-1, ::-1]
outshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:],
node.op.subsample, node.op.border_mode)
fulloutshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:], (1, 1),
node.op.border_mode)
nkern = op_imshp[1]
imshp = (op_kshp[0], outshp[0], outshp[1])
imshp_logical = (op_kshp[0], fulloutshp[0], fulloutshp[1])
din = ConvOp(imshp,
op_kshp[2:],
nkern,
op_imshp[0],
1,
1,
output_mode=mode,
unroll_batch=None,
unroll_kern=None,
unroll_patch=None,
imshp_logical=imshp_logical,
kshp_logical=None,
version=-1,
direction_hint='bprop inputs')
din = din(topgrad, filters)
din = patternbroadcast(din, node.outputs[0].broadcastable)
return [din]
def local_conv2d_gradweight_cpu(node):
img, topgrad, shape = node.inputs
if ((not isinstance(img.type, TensorType)
or not isinstance(topgrad.type, TensorType))):
return None
if node.op.border_mode not in ['full', 'valid']:
return None
if not node.op.filter_flip:
# Not tested yet
return
if node.op.border_mode == 'valid' and \
(node.op.subsample != (1, 1)):
# Use the gradient as defined in conv3D, because the implementation
# by Conv is slow (about 3x slower than conv3D, and probably 10x
# slower than it could be), nad incorrect when subsample > 2.
# build a "node", that should be equivalent to the one given by
# self.make_node, but using convGrad3D instead.
shuffled_img = img.dimshuffle(0, 2, 3, 'x', 1)
shuffled_topgrad = topgrad.dimshuffle(0, 2, 3, 'x', 1)
rval = convGrad3D(V=shuffled_img,
d=(node.op.subsample[0], node.op.subsample[1], 1),
WShape=(shuffled_topgrad.shape[4], shape[0],
shape[1], 1, shuffled_img.shape[4]),
dCdH=shuffled_topgrad)
rval = theano.tensor.addbroadcast(rval, 3)
rval = rval.dimshuffle(0, 4, 1, 2)
rval = rval[:, :, ::-1, ::-1]
rval = patternbroadcast(rval, node.outputs[0].broadcastable)
return [rval]
dx, dy = node.op.subsample
if dx not in (1, 2) or dy not in (1, 2):
# Not implemented in the gradient of ConvOp
return None
if node.op.imshp is None:
op_imshp = (None, None, None, None)
else:
op_imshp = node.op.imshp
if node.op.kshp is None:
op_kshp = (None, None, None, None)
else:
op_kshp = node.op.kshp
if None in op_imshp or None in op_kshp:
if (dx, dy) != (1, 1):
# We cannot infer the shapes
return None
# Determine gradient on kernels
assert len(op_imshp) == 4 and len(op_kshp) == 4
outshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:],
node.op.subsample, node.op.border_mode)
fulloutshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:], (1, 1),
node.op.border_mode)
newimg = img.dimshuffle((1, 0, 2, 3))
newtopgrad = topgrad.dimshuffle((1, 0, 2, 3))
if node.op.border_mode == 'valid':
(img, filters) = (newimg, newtopgrad)
kshp_logical = fulloutshp
kshp_logical_top_aligned = False
imshp_logical = None
(bsize, nkern) = (op_imshp[1], op_kshp[0])
imshp = (op_imshp[0], op_imshp[2], op_imshp[3])
kshp = outshp
elif node.op.border_mode == 'full':
(img, filters) = (newtopgrad, newimg)
kshp_logical = None
kshp_logical_top_aligned = True
imshp_logical = (op_imshp[0], fulloutshp[0], fulloutshp[1])
(bsize, nkern) = (op_kshp[0], op_imshp[1])
imshp = (op_imshp[0], outshp[0], outshp[1])
kshp = op_imshp[2:]
else:
raise NotImplementedError(
'Only [full,valid] modes are currently supported.')
# Flip the kernels
filters = filters[:, :, ::-1, ::-1]
dw = ConvOp(imshp,
kshp,
nkern,
bsize,
1,
1,
output_mode='valid',
unroll_batch=None,
unroll_kern=None,
unroll_patch=None,
imshp_logical=imshp_logical,
kshp_logical=kshp_logical,
kshp_logical_top_aligned=kshp_logical_top_aligned,
direction_hint='bprop weights')
res = dw(img, filters)
if node.op.border_mode == 'valid':
res = res.dimshuffle((1, 0, 2, 3))
res = res[:, :, ::-1, ::-1]
res = patternbroadcast(res, node.outputs[0].broadcastable)
return [res]
def __init__(self, inputs=None, params=None, outdir='outputs/conv1d',
n_filters=None, filter_size=None, stride=None, border_mode='valid',
weights_init='uniform', weights_interval='montreal', weights_mean=0, weights_std=5e-3,
bias_init=0,
activation='rectifier',
convolution='mc0',
mrg=RNG_MRG.MRG_RandomStreams(1),
**kwargs):
"""
Initialize a 1-D convolutional layer.
Parameters
----------
inputs : tuple(shape, `Theano.TensorType`)
The dimensionality of the inputs for this model, and the routing information for the model
to accept inputs from elsewhere. `shape` will be a monad tuple representing known
sizes for each dimension in the `Theano.TensorType`. Shape of the incoming data:
(batch_size, num_channels, data_dimensionality). Most likely, your channels
will be 1. For example, batches of text will be of the form (N, 1, D) where N=examples in minibatch and
D=dimensionality (chars, words, etc.)
params : Dict(string_name: theano SharedVariable), optional
A dictionary of model parameters (shared theano variables) that you should use when constructing
this model (instead of initializing your own shared variables). This parameter is useful when you want to
have two versions of the model that use the same parameters - such as siamese networks or pretraining some
weights.
outdir : str
The directory you want outputs (parameters, images, etc.) to save to. If None, nothing will
be saved.
n_filters : int
The number of filters to use (convolution kernels).
filter_size : int
The size of the convolution filter.
stride : int
The distance between the receptive field centers of neighboring units. This is the 'stride' of the
convolution operation.
border_mode : str, one of 'valid', 'full', 'same'
A string indicating the convolution border mode.
If 'valid', the convolution is only computed where the input and the
filter fully overlap.
If 'full', the convolution is computed wherever the input and the
filter overlap by at least one position.
weights_init : str
Determines the method for initializing model weights. See opendeep.utils.nnet for options.
weights_interval : str or float
If Uniform `weights_init`, the +- interval to use. See opendeep.utils.nnet for options.
weights_mean : float
If Gaussian `weights_init`, the mean value to use.
weights_std : float
If Gaussian `weights_init`, the standard deviation to use.
bias_init : float
The initial value to use for the bias parameter. Most often, the default of 0.0 is preferred.
activation : str or Callable
The activation function to apply to the layer. See opendeep.utils.activation for options.
convolution : str or Callable
The 1-dimensional convolution implementation to use. The default of 'mc0' is normally fine. See
opendeep.utils.conv1d_implementations for alternatives. (This is necessary because Theano only
supports 2D convolutions at the moment).
mrg : random
A random number generator that is used when adding noise.
I recommend using Theano's sandbox.rng_mrg.MRG_RandomStreams.
Notes
-----
Theano's default convolution function (`theano.tensor.nnet.conv.conv2d`)
does not support the 'same' border mode by default. This layer emulates
it by performing a 'full' convolution and then cropping the result, which
may negatively affect performance.
"""
initial_parameters = locals().copy()
initial_parameters.pop('self')
super(Conv1D, self).__init__(**initial_parameters)
if self.inputs is None:
return
##################
# specifications #
##################
# grab info from the inputs_hook, or from parameters
# expect input to be in the form (B, C, I) (batch, channel, input data)
# inputs_hook is a tuple of (Shape, Input)
# self.inputs is a list of all the input expressions (we enforce only 1, so self.inputs[0] is the input)
input_shape, self.input = self.inputs[0]
assert self.input.ndim == 3, "Expected 3D input variable with form (batch, channel, input_data)"
assert len(input_shape) == 3, "Expected 3D input shape with form (batch, channel, input_data)"
n_channels = input_shape[1]
filter_shape = (n_filters, n_channels, filter_size)
# activation function!
activation_func = get_activation_function(activation)
# convolution function!
convolution_func = get_conv1d_function(convolution)
outshape = ConvOp.getOutputShape(
inshp=(input_shape[-1],),
kshp=(filter_size,),
stride=(stride,),
mode=border_mode
)
self.output_size = (input_shape[0], n_filters) + outshape
##########
# Params #
##########
W = self.params.get(
"W",
get_weights(weights_init=weights_init,
shape=filter_shape,
name="W",
rng=mrg,
# if gaussian
mean=weights_mean,
std=weights_std,
# if uniform
interval=weights_interval)
)
b = self.params.get(
"b",
get_bias(shape=(n_filters,), name="b", init_values=bias_init)
)
# Finally have the two parameters!
self.params = OrderedDict([("W", W), ("b", b)])
########################
# Computational Graph! #
########################
if border_mode in ['valid', 'full']:
conved = convolution_func(self.input,
W,
subsample=(stride,),
image_shape=input_shape,
filter_shape=filter_shape,
border_mode=border_mode)
else:
log.error("Invalid border mode: '%s'" % border_mode)
raise RuntimeError("Invalid border mode: '%s'" % border_mode)
self.output = activation_func(conved + b.dimshuffle('x', 0, 'x'))
def __init__(self, inputs=None, params=None, outdir='outputs/conv2d',
n_filters=None, filter_size=None, stride=(1, 1), border_mode='valid',
weights_init='uniform', weights_interval='montreal', weights_mean=0, weights_std=5e-3,
bias_init=0,
activation='rectifier',
convolution='conv2d',
mrg=RNG_MRG.MRG_RandomStreams(1),
**kwargs):
"""
Initialize a 2-dimensional convolutional layer.
Parameters
----------
inputs : tuple(shape, `Theano.TensorType`)
The dimensionality of the inputs for this model, and the routing information for the model
to accept inputs from elsewhere. `shape` will be a monad tuple representing known
sizes for each dimension in the `Theano.TensorType`. Shape of the incoming data:
(batch_size, num_channels, input_height, input_width).
If input_size is None, it can be inferred. However, border_mode can't be 'same'.
params : Dict(string_name: theano SharedVariable), optional
A dictionary of model parameters (shared theano variables) that you should use when constructing
this model (instead of initializing your own shared variables). This parameter is useful when you want to
have two versions of the model that use the same parameters - such as siamese networks or pretraining some
weights.
outdir : str
The directory you want outputs (parameters, images, etc.) to save to. If None, nothing will
be saved.
n_filters : int
The number of filters to use (convolution kernels).
filter_size : tuple(int) or int
(filter_height, filter_width). If it is an int, size will be duplicated across height and width.
stride : tuple(int)
The distance between the receptive field centers of neighboring units. This is the 'stride' of the
convolution operation.
border_mode : str, one of 'valid', 'full'
A string indicating the convolution border mode.
If 'valid', the convolution is only computed where the input and the
filter fully overlap.
If 'full', the convolution is computed wherever the input and the
filter overlap by at least one position.
weights_init : str
Determines the method for initializing model weights. See opendeep.utils.nnet for options.
weights_interval : str or float
If Uniform `weights_init`, the +- interval to use. See opendeep.utils.nnet for options.
weights_mean : float
If Gaussian `weights_init`, the mean value to use.
weights_std : float
If Gaussian `weights_init`, the standard deviation to use.
bias_init : float
The initial value to use for the bias parameter. Most often, the default of 0.0 is preferred.
activation : str or Callable
The activation function to apply to the layer. See opendeep.utils.activation for options.
convolution : str or Callable
The 2-dimensional convolution implementation to use. The default of 'conv2d' is normally fine because it
uses theano's tensor.nnet.conv.conv2d, which cherry-picks the best implementation with a meta-optimizer if
you set the theano configuration flag 'optimizer_including=conv_meta'. Otherwise, you could pass a
callable function, such as cudnn or cuda-convnet if you don't want to use the meta-optimizer.
mrg : random
A random number generator that is used when adding noise.
I recommend using Theano's sandbox.rng_mrg.MRG_RandomStreams.
Notes
-----
Theano's default convolution function (`theano.tensor.nnet.conv.conv2d`)
does not support the 'same' border mode by default. This layer emulates
it by performing a 'full' convolution and then cropping the result, which
may negatively affect performance.
"""
super(Conv2D, self).__init__(**{arg: val for (arg, val) in locals().items() if arg is not 'self'})
##################
# specifications #
##################
# expect input to be in the form (B, C, 0, 1) (batch, channel, rows, cols)
# self.inputs is a list of all the input expressions (we enforce only 1, so self.inputs[0] is the input)
input_shape, self.input = self.inputs[0]
assert self.input.ndim == 4, "Expected 4D input variable with form (batch, channel, rows, cols)"
assert len(input_shape) == 4, "Expected 4D input shape with form (batch, channel, rows, cols)"
n_channels = input_shape[1]
if isinstance(filter_size, int):
filter_size = (filter_size, )*2
# activation function!
activation_func = get_activation_function(activation)
# convolution function!
if convolution == 'conv2d':
# using the theano flag optimizer_including=conv_meta will let this conv function optimize itself.
convolution_func = conv2d
else:
assert callable(convolution), "Input convolution was not 'conv2d' and was not Callable."
convolution_func = convolution
# filter shape should be in the form (num_filters, num_channels, filter_size[0], filter_size[1])
outshape = ConvOp.getOutputShape(
inshp=input_shape[-2:],
kshp=filter_size,
stride=stride,
mode=border_mode
)
self.output_size = (input_shape[0], n_filters) + outshape
filter_shape = (n_filters, n_channels) + filter_size
##########
# Params #
##########
W = self.params.get(
"W",
get_weights(weights_init=weights_init,
shape=filter_shape,
name="W",
rng=mrg,
# if gaussian
mean=weights_mean,
std=weights_std,
# if uniform
interval=weights_interval)
)
b = self.params.get(
"b",
get_bias(shape=(n_filters, ), name="b", init_values=bias_init)
)
# Finally have the two parameters!
self.params = OrderedDict([("W", W), ("b", b)])
########################
# Computational Graph! #
########################
if border_mode in ['valid', 'full']:
conved = convolution_func(self.input,
W,
subsample=stride,
image_shape=input_shape,
filter_shape=filter_shape,
border_mode=border_mode)
else:
raise RuntimeError("Invalid border mode: '%s'" % border_mode)
self.output = activation_func(conved + b.dimshuffle('x', 0, 'x', 'x'))
def local_conv2d_gradinputs_cpu(node):
kern, topgrad, shape = node.inputs
if ((not isinstance(kern.type, TensorType) or
not isinstance(topgrad.type, TensorType))):
return None
if node.op.border_mode not in ['full', 'valid']:
return None
if not node.op.filter_flip:
# Not tested yet
return None
# Conv 3d implementation, needed when subsample > 2
if node.op.border_mode == 'valid' and node.op.subsample != (1, 1):
kern = kern[:, :, ::-1, ::-1]
shuffled_kern = kern.dimshuffle(0, 2, 3, 'x', 1)
shuffled_topgrad = topgrad.dimshuffle(0, 2, 3, 'x', 1)
b = theano.tensor.zeros_like(shuffled_kern[0, 0, 0, 0, :])
rval = convTransp3D(W=shuffled_kern, b=b,
d=(node.op.subsample[0], node.op.subsample[1], 1),
H=shuffled_topgrad,
RShape=(shape[0], shape[1], 1))
rval = theano.tensor.addbroadcast(rval, 3)
rval = rval.dimshuffle(0, 4, 1, 2)
rval = patternbroadcast(rval, node.outputs[0].broadcastable)
return [rval]
# Conv2d Implementation
dx, dy = node.op.subsample
if dx not in (1, 2) or dy not in (1, 2):
# Not implemented in the gradient of ConvOp
return None
if node.op.imshp is None:
op_imshp = (None, None, None, None)
else:
op_imshp = node.op.imshp
if node.op.kshp is None:
op_kshp = (None, None, None, None)
else:
op_kshp = node.op.kshp
if None in op_imshp or None in op_kshp:
if (dx, dy) != (1, 1):
return None
mode = 'valid'
if not node.op.border_mode == 'full':
mode = 'full'
filters = kern.dimshuffle((1, 0, 2, 3))
filters = filters[:, :, ::-1, ::-1]
outshp = ConvOp.getOutputShape(op_imshp[2:],
op_kshp[2:], node.op.subsample,
node.op.border_mode)
fulloutshp = ConvOp.getOutputShape(op_imshp[2:],
op_kshp[2:], (1, 1),
node.op.border_mode)
nkern = op_imshp[1]
imshp = (op_kshp[0], outshp[0], outshp[1])
imshp_logical = (op_kshp[0], fulloutshp[0], fulloutshp[1])
din = ConvOp(imshp,
op_kshp[2:],
nkern,
op_imshp[0],
1, 1, output_mode=mode,
unroll_batch=None, unroll_kern=None,
unroll_patch=None,
imshp_logical=imshp_logical,
kshp_logical=None,
version=-1,
direction_hint='bprop inputs')
din = din(topgrad, filters)
din = patternbroadcast(din, node.outputs[0].broadcastable)
return [din]
def local_conv2d_gradweight_cpu(node):
img, topgrad, shape = node.inputs
if ((not isinstance(img.type, TensorType) or
not isinstance(topgrad.type, TensorType))):
return None
if node.op.border_mode not in ['full', 'valid']:
return None
if not node.op.filter_flip:
# Not tested yet
return
if node.op.border_mode == 'valid' and \
(node.op.subsample != (1, 1)):
# Use the gradient as defined in conv3D, because the implementation
# by Conv is slow (about 3x slower than conv3D, and probably 10x
# slower than it could be), nad incorrect when subsample > 2.
# build a "node", that should be equivalent to the one given by
# self.make_node, but using convGrad3D instead.
shuffled_img = img.dimshuffle(0, 2, 3, 'x', 1)
shuffled_topgrad = topgrad.dimshuffle(0, 2, 3, 'x', 1)
rval = convGrad3D(V=shuffled_img,
d=(node.op.subsample[0], node.op.subsample[1], 1),
WShape=(shuffled_topgrad.shape[4],
shape[0], shape[1], 1,
shuffled_img.shape[4]),
dCdH=shuffled_topgrad)
rval = theano.tensor.addbroadcast(rval, 3)
rval = rval.dimshuffle(0, 4, 1, 2)
rval = rval[:, :, ::-1, ::-1]
rval = patternbroadcast(rval, node.outputs[0].broadcastable)
return [rval]
dx, dy = node.op.subsample
if dx not in (1, 2) or dy not in (1, 2):
# Not implemented in the gradient of ConvOp
return None
if node.op.imshp is None:
op_imshp = (None, None, None, None)
else:
op_imshp = node.op.imshp
if node.op.kshp is None:
op_kshp = (None, None, None, None)
else:
op_kshp = node.op.kshp
if None in op_imshp or None in op_kshp:
if (dx, dy) != (1, 1):
# We cannot infer the shapes
return None
# Determine gradient on kernels
assert len(op_imshp) == 4 and len(op_kshp) == 4
outshp = ConvOp.getOutputShape(op_imshp[2:],
op_kshp[2:], node.op.subsample,
node.op.border_mode)
fulloutshp = ConvOp.getOutputShape(op_imshp[2:],
op_kshp[2:], (1, 1),
node.op.border_mode)
newimg = img.dimshuffle((1, 0, 2, 3))
newtopgrad = topgrad.dimshuffle((1, 0, 2, 3))
if node.op.border_mode == 'valid':
(img, filters) = (newimg, newtopgrad)
kshp_logical = fulloutshp
kshp_logical_top_aligned = False
imshp_logical = None
(bsize, nkern) = (op_imshp[1], op_kshp[0])
imshp = (op_imshp[0], op_imshp[2], op_imshp[3])
kshp = outshp
elif node.op.border_mode == 'full':
(img, filters) = (newtopgrad, newimg)
kshp_logical = None
kshp_logical_top_aligned = True
imshp_logical = (op_imshp[0],
fulloutshp[0],
fulloutshp[1])
(bsize, nkern) = (op_kshp[0], op_imshp[1])
imshp = (op_imshp[0], outshp[0], outshp[1])
kshp = op_imshp[2:]
else:
raise NotImplementedError(
'Only [full,valid] modes are currently supported.')
# Flip the kernels
filters = filters[:, :, ::-1, ::-1]
dw = ConvOp(imshp, kshp, nkern, bsize, 1, 1, output_mode='valid',
unroll_batch=None, unroll_kern=None, unroll_patch=None,
imshp_logical=imshp_logical,
kshp_logical=kshp_logical,
kshp_logical_top_aligned=kshp_logical_top_aligned,
direction_hint='bprop weights')
res = dw(img, filters)
if node.op.border_mode == 'valid':
res = res.dimshuffle((1, 0, 2, 3))
res = res[:, :, ::-1, ::-1]
res = patternbroadcast(res, node.outputs[0].broadcastable)
return [res]