def run_conv_nnet1(use_gpu):
if use_gpu:
shared_fn = tcn.shared_constructor
else:
shared_fn = shared
n_batch = 16
n_kern = 20
shape_img = (n_batch, 1, 32, 32)
shape_kern = (n_kern, 1, 5, 5)
n_train = 10
if config.mode == 'DEBUG_MODE':
n_train = 1
logical_hid_shape = tcn.blas.GpuConv.logical_output_shape_2d(
shape_img[2:], shape_kern[2:], 'valid')
n_hid = n_kern * logical_hid_shape[0] * logical_hid_shape[1]
n_out = 10
w = shared_fn(0.01 * (my_rand(*shape_kern) - 0.5), 'w')
b = shared_fn(my_zeros((n_kern,)), 'b')
v = shared_fn(my_zeros((n_hid, n_out)), 'c')
c = shared_fn(my_zeros(n_out), 'c')
x = tensor.Tensor(dtype='float32', broadcastable=(0, 1, 0, 0))('x')
y = tensor.fmatrix('y')
lr = tensor.fscalar('lr')
conv_op = conv.ConvOp(shape_img[2:], shape_kern[2:], n_kern, n_batch, 1, 1)
hid = tensor.tanh(conv_op(x, w) + b.dimshuffle((0, 'x', 'x')))
hid_flat = hid.reshape((n_batch, n_hid))
out = tensor.tanh(tensor.dot(hid_flat, v) + c)
loss = tensor.sum(0.5 * (out - y) ** 2 * lr)
# print 'loss type', loss.type
params = [w, b, v, c]
gparams = tensor.grad(loss, params)
mode = get_mode(use_gpu)
# print 'building pfunc ...'
train = pfunc(
[x, y, lr],
[loss],
mode=mode,
updates=[(p, p - g) for p, g in zip(params, gparams)])
# for i, n in enumerate(train.maker.fgraph.toposort()):
# print i, n
xval = my_rand(*shape_img)
yval = my_rand(n_batch, n_out)
lr = theano._asarray(0.01, dtype='float32')
for i in xrange(n_train):
rval = train(xval, yval, lr)
# print 'training done'
print_mode(mode)
return rval
def build_conv_nnet2_classif(use_gpu, isize, ksize, n_batch,
downsample_ops=True, verbose=0, version=-1,
check_isfinite=True):
if use_gpu:
shared_fn = tcn.shared_constructor
else:
shared_fn = shared
isize1 = isize
isize2 = isize
if isinstance(isize, (tuple, )):
isize1 = isize[0]
isize2 = isize[1]
shape_img = (n_batch, 1, isize1, isize2)
n_kern = 20 # 6 were used in LeNet5
shape_kern = (n_kern, 1, ksize, ksize)
n_kern1 = 30 # 16 were used in LeNet5
shape_kern1 = (n_kern1, n_kern, ksize, ksize)
logical_hid_shape = tcn.blas.GpuConv.logical_output_shape_2d(
(isize1, isize2), (ksize, ksize), 'valid')
logical_hid_shape1 = tcn.blas.GpuConv.logical_output_shape_2d(
(logical_hid_shape[0] // 2, logical_hid_shape[1] // 2),
(ksize, ksize), 'valid')
n_hid = n_kern1 * logical_hid_shape1[0] * logical_hid_shape1[1]
n_out = 10
w0 = shared_fn(0.01 * (my_rand(*shape_kern) - 0.5), 'w0')
b0 = shared_fn(my_zeros((n_kern,)), 'b0')
w1 = shared_fn(0.01 * (my_rand(*shape_kern1) - 0.5), 'w1')
b1 = shared_fn(my_zeros((n_kern1,)), 'b1')
v = shared_fn(0.01 * my_randn(n_hid, n_out), 'v')
c = shared_fn(my_zeros(n_out), 'c')
# print 'ALLOCATING ARCH: w0 shape', w0.get_value(borrow=True).shape
# print 'ALLOCATING ARCH: w1 shape', w1.get_value(borrow=True).shape
# print 'ALLOCATING ARCH: v shape', v.get_value(borrow=True).shape
x = tensor.Tensor(dtype='float32', broadcastable=(0, 1, 0, 0))('x')
y = tensor.fmatrix('y')
lr = tensor.fscalar('lr')
conv_op = conv.ConvOp(shape_img[2:], shape_kern[2:], n_kern,
n_batch, 1, 1, verbose=verbose, version=version)
conv_op1 = conv.ConvOp(
(n_kern, logical_hid_shape[0] // 2, logical_hid_shape[1] // 2),
shape_kern1[2:], n_kern1, n_batch, 1, 1, verbose=verbose, version=version)
ds_op = pool.Pool((2, 2), ignore_border=False)
if downsample_ops:
hid = tensor.tanh(ds_op(conv_op(x, w0) + b0.dimshuffle((0, 'x', 'x'))))
else:
hid = tensor.tanh(
(conv_op(x, w0) + b0.dimshuffle(
(0, 'x', 'x')))[:, :, ::2, ::2])
hid1 = tensor.tanh(conv_op1(hid, w1) + b1.dimshuffle((0, 'x', 'x')))
hid_flat = hid1.reshape((n_batch, n_hid))
out = tensor.nnet.softmax(tensor.dot(hid_flat, v) + c)
loss = tensor.sum(tensor.nnet.crossentropy_categorical_1hot(
out, tensor.argmax(y, axis=1)) * lr)
# print 'loss type', loss.type
params = [w0, b0, w1, b1, v, c]
gparams = tensor.grad(loss, params)
mode = get_mode(use_gpu, check_isfinite)
# print 'building pfunc ...'
train = pfunc(
[x, y, lr],
[loss],
mode=mode,
updates=[(p, p - g) for p, g in zip(params, gparams)])
if verbose:
theano.printing.debugprint(train)
if use_gpu:
# Check that GpuConv is used
topo = train.maker.fgraph.toposort()
conv_ops = (tcn.blas.GpuConv,
tcn.dnn.GpuDnnConv,
tcn.dnn.GpuDnnConvGradI,
tcn.dnn.GpuDnnConvGradW,
tcn.blas.BaseGpuCorrMM)
assert len([n for n in topo if isinstance(n.op, conv_ops)]) > 0
shape_target = (n_batch, n_out)
return train, params, shape_img, shape_target, mode
def run_conv_nnet2(use_gpu): # pretend we are training LeNet for MNIST
if use_gpu:
shared_fn = tcn.shared_constructor
else:
shared_fn = shared
# cumulativ rounding error affect this comparaison of result. So we lower the tolerance.
# TODO: why the last two example see the error lower? We are converging?
# n_train=10, n_batch=3, n_kern=1, n_kern1=1, error see of 1e-9
# n_train=10, n_batch=3, n_kern=10, n_kern1=1, error see of -1.27777e-06
# n_train=10, n_batch=3, n_kern=10, n_kern1=10, error see of -6.91377e-05
# n_train=10, n_batch=30, n_kern=10, n_kern1=10, error see of -0.00185963
# n_train=10, n_batch=60, n_kern=10, n_kern1=10, error see of -5.26905e-05
# n_train=30, n_batch=60, n_kern=10, n_kern1=10, error see of -3.8147e-06
# n_train=30, n_batch=60, n_kern=20, n_kern1=10, error see of 6.82771e-05
# n_train=30, n_batch=60, n_kern=20, n_kern1=30, error see of 0.000231534
n_batch = 60
shape_img = (n_batch, 1, 32, 32)
n_kern = 20
shape_kern = (n_kern, 1, 5, 5)
n_kern1 = 10
shape_kern1 = (n_kern1, n_kern, 5, 5)
n_train = 30
if config.mode == 'DEBUG_MODE':
n_train = 1
logical_hid_shape = tcn.blas.GpuConv.logical_output_shape_2d(tuple(
shape_img[2:]), tuple(shape_kern[2:]), 'valid')
logical_hid_shape1 = tcn.blas.GpuConv.logical_output_shape_2d(
(logical_hid_shape[0] // 2, logical_hid_shape[1] // 2),
tuple(shape_kern1[2:]), 'valid')
n_hid = n_kern1 * logical_hid_shape1[0] * logical_hid_shape1[1]
n_out = 10
w0 = shared_fn(0.01 * (my_rand(*shape_kern) - 0.5), 'w0')
b0 = shared_fn(my_zeros((n_kern,)), 'b0')
w1 = shared_fn(0.01 * (my_rand(*shape_kern1) - 0.5), 'w1')
b1 = shared_fn(my_zeros((n_kern1,)), 'b1')
v = shared_fn(my_zeros((n_hid, n_out)), 'c')
c = shared_fn(my_zeros(n_out), 'c')
x = tensor.Tensor(dtype='float32', broadcastable=(0, 1, 0, 0))('x')
y = tensor.fmatrix('y')
lr = tensor.fscalar('lr')
conv_op = conv.ConvOp(shape_img[2:], shape_kern[2:], n_kern, n_batch, 1, 1)
conv_op1 = conv.ConvOp((n_kern, logical_hid_shape[0] // 2,
logical_hid_shape[1] // 2),
shape_kern1[2:],
n_kern1, n_batch, 1, 1)
hid = tensor.tanh(conv_op(x, w0) + b0.dimshuffle((0, 'x', 'x')))
hid1 = tensor.tanh(conv_op1(hid[:, :, ::2, ::2], w1) + b1.dimshuffle((
0, 'x', 'x')))
hid_flat = hid1.reshape((n_batch, n_hid))
out = tensor.tanh(tensor.dot(hid_flat, v) + c)
loss = tensor.sum(0.5 * (out - y) ** 2 * lr)
# print 'loss type', loss.type
params = [w0, b0, w1, b1, v, c]
gparams = tensor.grad(loss, params)
mode = get_mode(use_gpu)
# print 'building pfunc ...'
train = pfunc(
[x, y, lr],
[loss],
mode=mode,
updates=[(p, p - g) for p, g in zip(params, gparams)])
# for i, n in enumerate(train.maker.fgraph.toposort()):
# print i, n
xval = my_rand(*shape_img)
yval = my_rand(n_batch, n_out) # int32 make all 0...
lr = theano._asarray(0.01, dtype='float32')
for i in xrange(n_train):
rval = train(xval, yval, lr)
print_mode(mode)
return rval
def conv2d(input,
filters,
image_shape=None,
filter_shape=None,
border_mode='valid',
subsample=(1, 1),
**kargs):
"""
signal.conv.conv2d performs a basic 2D convolution of the input with the
given filters. The input parameter can be a single 2D image or a 3D tensor,
containing a set of images. Similarly, filters can be a single 2D filter or
a 3D tensor, corresponding to a set of 2D filters.
Shape parameters are optional and will result in faster execution.
Parameters
----------
input : dmatrix of dtensor3
Symbolic variable for images to be filtered.
filters : dmatrix of dtensor3
Symbolic variable containing filter values.
border_mode: {'valid', 'full'}
See scipy.signal.convolve2d.
subsample
Factor by which to subsample output.
image_shape : tuple of length 2 or 3
([number images,] image height, image width).
filter_shape : tuple of length 2 or 3
([number filters,] filter height, filter width).
kwargs
See theano.tensor.nnet.conv.conv2d.
Returns
-------
symbolic 2D,3D or 4D tensor
Tensor of filtered images, with shape
([number images,] [number filters,] image height, image width).
"""
assert input.ndim in (2, 3)
assert filters.ndim in (2, 3)
# use shape information if it is given to us ###
if filter_shape and image_shape:
if input.ndim == 3:
bsize = image_shape[0]
else:
bsize = 1
imshp = (1, ) + tuple(image_shape[-2:])
if filters.ndim == 3:
nkern = filter_shape[0]
else:
nkern = 1
kshp = filter_shape[-2:]
else:
nkern, kshp = None, None
bsize, imshp = None, None
# reshape tensors to 4D, for compatibility with ConvOp ###
if input.ndim == 3:
sym_bsize = input.shape[0]
else:
sym_bsize = 1
if filters.ndim == 3:
sym_nkern = filters.shape[0]
else:
sym_nkern = 1
new_input_shape = tensor.join(0, tensor.stack([sym_bsize, 1]),
input.shape[-2:])
input4D = tensor.reshape(input, new_input_shape, ndim=4)
new_filter_shape = tensor.join(0, tensor.stack([sym_nkern, 1]),
filters.shape[-2:])
filters4D = tensor.reshape(filters, new_filter_shape, ndim=4)
# perform actual convolution ###
op = conv.ConvOp(output_mode=border_mode,
dx=subsample[0],
dy=subsample[1],
imshp=imshp,
kshp=kshp,
nkern=nkern,
bsize=bsize,
**kargs)
output = op(input4D, filters4D)
# flatten to 3D tensor if convolving with single filter or single image
if input.ndim == 2 and filters.ndim == 2:
if theano.config.warn.signal_conv2d_interface:
warnings.warn(
"theano.tensor.signal.conv2d() now outputs a 2d tensor when both"
" inputs are 2d. To disable this warning, set the Theano flag"
" warn.signal_conv2d_interface to False",
stacklevel=3)
output = tensor.flatten(output.T, outdim=2).T
elif input.ndim == 2 or filters.ndim == 2:
output = tensor.flatten(output.T, outdim=3).T
return output
def conv2d(input, filters, image_shape=None, filter_shape=None,
border_mode='valid', subsample=(1,1), **kargs):
"""
signal.conv.conv2d performs a basic 2D convolution of the input with the
given filters. The input parameter can be a single 2D image or a 3D tensor,
containing a set of images. Similarly, filters can be a single 2D filter or
a 3D tensor, corresponding to a set of 2D filters.
Shape parameters are optional and will result in faster execution.
:type input: dmatrix of dtensor3
:param input: symbolic variable for images to be filtered
:type filters: dmatrix of dtensor3
:param filters: symbolic variable containing filter values
:param border_mode: 'valid' or 'full'. see scipy.signal.convolve2d
:param subsample: factor by which to subsample output
:type image_shape: tuple of length 2 or 3
:param image_shape: ([number images,] image height, image width)
:type filter_shape: tuple of length 2 or 3
:param filter_shape: ([number filters,] filter height, filter width)
:param kwargs: see theano.tensor.nnet.conv.conv2d
:rtype: symbolic 2D,3D or 4D tensor
:return: tensor of filtered images, with shape
([number images,] [number filters,] image height, image width)
"""
assert input.ndim in (2,3)
assert filters.ndim in (2,3)
### use shape information if it is given to us ###
if filter_shape and image_shape:
if input.ndim==3:
bsize = image_shape[0]
else:
bsize = 1
imshp = (1,) + tuple(image_shape[-2:])
if filters.ndim==3:
nkern = filter_shape[0]
else:
nkern = 1
kshp = filter_shape[-2:]
else:
nkern, kshp = None, None
bsize, imshp = None, None
### reshape tensors to 4D, for compatibility with ConvOp ###
if input.ndim==3:
sym_bsize = input.shape[0]
else:
sym_bsize = 1
if filters.ndim==3:
sym_nkern = filters.shape[0]
else:
sym_nkern = 1
new_input_shape = tensor.join(0, tensor.stack(sym_bsize,1), input.shape[-2:])
input4D = tensor.reshape(input, new_input_shape, ndim=4)
new_filter_shape = tensor.join(0, tensor.stack(sym_nkern,1), filters.shape[-2:])
filters4D = tensor.reshape(filters, new_filter_shape, ndim=4)
### perform actual convolution ###
op = conv.ConvOp(output_mode=border_mode,
dx=subsample[0], dy=subsample[1],
imshp=imshp, kshp=kshp, nkern=nkern, bsize=bsize,**kargs)
output = op(input4D, filters4D)
# flatten to 3D tensor if convolving with single filter or single image
if input.ndim==2 or filters.ndim==2:
output = tensor.flatten(output.T, outdim=3).T
return output