def test_pooling_opt():
if not dnn.dnn_available(test_ctx_name):
raise SkipTest(dnn.dnn_available.msg)
x = T.fmatrix()
f = theano.function(
[x],
pool_2d(x, ds=(2, 2), mode='average_inc_pad',
ignore_border=True),
mode=mode_with_gpu)
assert any([isinstance(n.op, dnn.GpuDnnPool)
for n in f.maker.fgraph.toposort()])
f(numpy.zeros((10, 10), dtype='float32'))
f = theano.function(
[x],
T.grad(pool_2d(x, ds=(2, 2), mode='average_inc_pad',
ignore_border=True).sum(),
x),
mode=mode_with_gpu.including("cudnn"))
assert any([isinstance(n.op, dnn.GpuDnnPoolGrad)
for n in f.maker.fgraph.toposort()])
f(numpy.zeros((10, 10), dtype='float32'))
def test_old_pool_interface(self):
if sys.version_info[0] != 3:
# Only tested with python 3 because of pickling issues.
raise SkipTest('Skip old pool interface with python 2.x')
# 1. Load the old version
testfile_dir = os.path.dirname(os.path.realpath(__file__))
fname = 'old_pool_interface.pkl'
with open(os.path.join(testfile_dir, fname), 'rb') as fp:
try:
old_fct = cPickle.load(fp, encoding='latin1')
except ImportError:
# Windows sometimes fail with nonsensical errors like:
# ImportError: No module named type
# ImportError: No module named copy_reg
# when "type" and "copy_reg" are builtin modules.
if sys.platform == 'win32':
exc_type, exc_value, exc_trace = sys.exc_info()
reraise(SkipTest, exc_value, exc_trace)
raise
# 2. Create the new version
x = theano.tensor.ftensor4()
y = pool_2d(x, (2, 2), mode='max', ignore_border=True)
z = pool_2d(x, (2, 2), mode='average_exc_pad', ignore_border=True)
dy_dx = theano.gradient.grad(y.sum(), x)
dz_dx = theano.gradient.grad(z.sum(), x)
new_fct = theano.function([x], [y, z, dy_dx, dz_dx])
# 3. Assert that the answer is the same
rng = numpy.random.RandomState(utt.fetch_seed())
image_val = rng.rand(4, 6, 7, 9).astype(numpy.float32)
old_out = old_fct(image_val)
new_out = new_fct(image_val)
for o, n in zip(old_out, new_out):
utt.assert_allclose(o, n)
def test_pooling_with_tensor_vars(self):
x = tensor.ftensor4()
window_size = tensor.ivector()
stride = tensor.ivector()
padding = tensor.ivector()
data = numpy.random.normal(0, 1, (1, 1, 5, 5)).astype('float32')
# checking variable params vs fixed params
for ignore_border in [True, False]:
for mode in ['max', 'sum', 'average_inc_pad', 'average_exc_pad']:
y = pool_2d(x, window_size, ignore_border, stride, padding,
mode)
dx = theano.gradient.grad(y.sum(), x)
var_fct = theano.function([x, window_size, stride, padding],
[y, dx])
for ws in (4, 2, 5):
for st in (2, 3):
for pad in (0, 1):
if (pad > st or st > ws or
(pad != 0 and not ignore_border) or
(mode == 'average_exc_pad' and pad != 0)):
continue
y = pool_2d(x, (ws, ws), ignore_border, (st, st),
(pad, pad), mode)
dx = theano.gradient.grad(y.sum(), x)
fix_fct = theano.function([x], [y, dx])
var_y, var_dx = var_fct(data, (ws, ws), (st, st),
(pad, pad))
fix_y, fix_dx = fix_fct(data)
utt.assert_allclose(var_y, fix_y)
utt.assert_allclose(var_dx, fix_dx)
def model(X, params, featMaps, pieces, pDropConv, pDropHidden):
lnum = 0 # conv: (32, 32) pool: (16, 16)
layer = conv2d(X, params[lnum][0], border_mode='half') + \
params[lnum][1].dimshuffle('x', 0, 'x', 'x')
layer = maxout(layer, featMaps[lnum], pieces[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (16, 16) pool: (8, 8)
layer = conv2d(layer, params[lnum][0], border_mode='half') + \
params[lnum][1].dimshuffle('x', 0, 'x', 'x')
layer = maxout(layer, featMaps[lnum], pieces[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (8, 8) pool: (4, 4)
layer = conv2d(layer, params[lnum][0], border_mode='half') + \
params[lnum][1].dimshuffle('x', 0, 'x', 'x')
layer = maxout(layer, featMaps[lnum], pieces[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1
layer = T.flatten(layer, outdim=2)
layer = T.dot(layer, params[lnum][0]) + params[lnum][1].dimshuffle('x', 0)
layer = relu(layer, alpha=0)
layer = basicUtils.dropout(layer, pDropHidden)
lnum += 1
layer = T.dot(layer, params[lnum][0]) + params[lnum][1].dimshuffle('x', 0)
layer = relu(layer, alpha=0)
layer = basicUtils.dropout(layer, pDropHidden)
lnum += 1
return softmax(T.dot(layer, params[lnum][0]) + params[lnum][1].dimshuffle('x', 0)) # 如果使用nnet中的softmax训练产生NAN
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2), non_linear="tanh"):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: np.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps, filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps, image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows,#cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
self.filter_shape = filter_shape
self.image_shape = image_shape
self.poolsize = poolsize
self.non_linear = non_linear
self.output_shape = (image_shape[0],filter_shape[0],int((image_shape[2]-filter_shape[2]+1)/poolsize[0]),int(image_shape[3]-filter_shape[3]+1)/poolsize[1])
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = np.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) /np.prod(poolsize))
# initialize weights with random weights
if self.non_linear=="none" or self.non_linear=="relu":
self.W = theano.shared(np.asarray(rng.uniform(low=-0.01,high=0.01,size=filter_shape),
dtype=theano.config.floatX),borrow=True,name="W_conv")
else:
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(np.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX),borrow=True,name="W_conv")
b_values = np.zeros((self.output_shape[1],image_shape[2]-filter_shape[2]+1,image_shape[3]-filter_shape[3]+1), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True, name="b_conv")
# convolve input feature maps with filters
self.conv_out = conv.conv2d(input=input, filters=self.W,filter_shape=self.filter_shape, image_shape=self.image_shape)
if self.non_linear=="tanh":
self.conv_out_tanh = T.tanh(self.conv_out + self.b)
self.output = pool.pool_2d(input=self.conv_out_tanh, ds=self.poolsize, ignore_border=True)
elif self.non_linear=="relu":
self.conv_out_tanh = ReLU(self.conv_out + self.b)
self.output = pool.pool_2d(input=self.conv_out_tanh, ds=self.poolsize, ignore_border=True)
else:
pooled_out = pool.pool_2d(input=self.conv_out, ds=self.poolsize, ignore_border=True)
self.output = pooled_out + self.b
self.params = [self.W, self.b]
self.L2 = (self.W**2).sum()
def CNN(x,c_l1,c_l2,f_l1,f_l2,insize):
print "in size ", insize
conv1=tensor.nnet.relu(conv2d(x,c_l1)) #default stride=1 --subsample=(1,1)
conv1_shp=get_conv_output_shape(insize,c_l1.get_value().shape,border_mode='valid',subsample=(1,1))
print "conv1 size ", conv1_shp
pool1=pool_2d(conv1,(3,3),st=(3,3),ignore_border=True) #default maxpool
pool1_shp=get_pool_output_shape(conv1_shp,pool_size=(3,3),st=(3,3),ignore_border=True)
print "pool1 size ", pool1_shp
lrn1=LRN(pool1,pool1_shp)
lrn1_shp=tuple(pool1_shp)
print "cross map norm1 size ", lrn1_shp
conv2=tensor.nnet.relu(conv2d(lrn1,c_l2))
conv2_shp=get_conv_output_shape(lrn1_shp,c_l2.get_value().shape,border_mode='valid',subsample=(1,1))
print "conv2 size ", conv2_shp
pool2=pool_2d(conv2,(2,2),st=(2,2),ignore_border=True)
pool2_shp=get_pool_output_shape(conv2_shp,pool_size=(2,2),st=(2,2),ignore_border=True)
print "pool2 size ", pool2_shp
lrn2=LRN(pool2,pool2_shp)
lrn2_shp=tuple(pool2_shp)
print "cross map norm2 size " , lrn2_shp
fpool2=tensor.flatten(lrn2,outdim=2)
full1=tensor.nnet.relu(tensor.dot(fpool2,f_l1))
pyx=tensor.nnet.sigmoid(tensor.dot(full1,f_l2))
return c_l1, c_l2, f_l1, f_l2, pyx
def pool2d(x, pool_size, strides=(1, 1), border_mode="valid", dim_ordering=_IMAGE_DIM_ORDERING, pool_mode="max"):
if border_mode == "same":
w_pad = pool_size[0] - 2 if pool_size[0] % 2 == 1 else pool_size[0] - 1
h_pad = pool_size[1] - 2 if pool_size[1] % 2 == 1 else pool_size[1] - 1
padding = (w_pad, h_pad)
elif border_mode == "valid":
padding = (0, 0)
else:
raise Exception("Invalid border mode: " + str(border_mode))
if dim_ordering not in {"th", "tf"}:
raise Exception("Unknown dim_ordering " + str(dim_ordering))
if dim_ordering == "tf":
x = x.dimshuffle((0, 3, 1, 2))
if pool_mode == "max":
pool_out = pool.pool_2d(x, ds=pool_size, st=strides, ignore_border=True, padding=padding, mode="max")
elif pool_mode == "avg":
pool_out = pool.pool_2d(
x, ds=pool_size, st=strides, ignore_border=True, padding=padding, mode="average_exc_pad"
)
else:
raise Exception("Invalid pooling mode: " + str(pool_mode))
if border_mode == "same":
expected_width = (x.shape[2] + strides[0] - 1) // strides[0]
expected_height = (x.shape[3] + strides[1] - 1) // strides[1]
pool_out = pool_out[:, :, :expected_width, :expected_height]
if dim_ordering == "tf":
pool_out = pool_out.dimshuffle((0, 2, 3, 1))
return pool_out
def model(X, params, pDropConv, pDropHidden):
lnum = 0 # conv: (32, 32) pool: (16, 16)
layer = nin(X, params[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (16, 16) pool: (8, 8)
layer = nin(layer, params[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (8, 8) pool: (4, 4)
layer = nin(layer, params[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
# 全连接层
# layer = T.flatten(layer, outdim=2)
# lnum += 1
# layer = fc(layer, params[lnum])
# layer = utils.dropout(layer, pDropHidden)
# lnum += 1
# layer = fc(layer, params[lnum])
# 全局平均池化
lnum += 1
layer = conv1t1(layer, params[lnum])
layer = basicUtils.dropout(layer, pDropHidden)
lnum += 1
layer = conv1t1(layer, params[lnum])
layer = gap(layer)
return softmax(layer) # 如果使用nnet中的softmax训练产生NAN
def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
dim_ordering='th', pool_mode='max'):
if border_mode == 'same':
# TODO: add implementation for border_mode="same"
raise Exception('border_mode="same" not supported with Theano.')
elif border_mode == 'valid':
ignore_border = True
padding = (0, 0)
else:
raise Exception('Invalid border mode: ' + str(border_mode))
if dim_ordering not in {'th', 'tf'}:
raise Exception('Unknown dim_ordering ' + str(dim_ordering))
if dim_ordering == 'tf':
x = x.dimshuffle((0, 4, 1, 2, 3))
if pool_mode == 'max':
# pooling over conv_dim2, conv_dim1 (last two channels)
output = pool.pool_2d(input=x.dimshuffle(0, 1, 4, 3, 2),
ds=(pool_size[1], pool_size[0]),
st=(strides[1], strides[0]),
ignore_border=ignore_border,
padding=padding,
mode='max')
# pooling over conv_dim3
pool_out = pool.pool_2d(input=output.dimshuffle(0, 1, 4, 3, 2),
ds=(1, pool_size[2]),
st=(1, strides[2]),
ignore_border=ignore_border,
padding=padding,
mode='max')
elif pool_mode == 'avg':
# pooling over conv_dim2, conv_dim1 (last two channels)
output = pool.pool_2d(input=x.dimshuffle(0, 1, 4, 3, 2),
ds=(pool_size[1], pool_size[0]),
st=(strides[1], strides[0]),
ignore_border=ignore_border,
padding=padding,
mode='average_exc_pad')
# pooling over conv_dim3
pool_out = pool.pool_2d(input=output.dimshuffle(0, 1, 4, 3, 2),
ds=(1, pool_size[2]),
st=(1, strides[2]),
ignore_border=ignore_border,
padding=padding,
mode='average_exc_pad')
else:
raise Exception('Invalid pooling mode: ' + str(pool_mode))
if dim_ordering == 'tf':
pool_out = pool_out.dimshuffle((0, 2, 3, 4, 1))
return pool_out
def CNN(x,c_l1,c_l2,f_l1,f_l2):
conv1=tensor.nnet.relu(conv2d(x,c_l1)) #default stride=1 --subsample=(1,1)
pool1=pool_2d(conv1,(2,2),st=(2,2),ignore_border=True) #default maxpool
conv2=tensor.nnet.relu(conv2d(pool1,c_l2))
pool2=pool_2d(conv2,(2,2),st=(2,2),ignore_border=True)
fpool2=tensor.flatten(pool2,outdim=2)
full1=tensor.nnet.relu(tensor.dot(fpool2,f_l1))
pyx=tensor.nnet.sigmoid(tensor.dot(full1,f_l2))
return c_l1, c_l2, f_l1, f_l2, pyx
def modelFlow(X, params):
lconv1 = relu(conv2d(X, params[0][0], border_mode='full') +
params[0][1].dimshuffle('x', 0, 'x', 'x'))
lds1 = pool_2d(lconv1, (2, 2))
lconv2 = relu(conv2d(lds1, params[1][0]) +
params[1][1].dimshuffle('x', 0, 'x', 'x'))
lds2 = pool_2d(lconv2, (2, 2))
lconv3 = relu(conv2d(lds2, params[2][0]) +
params[2][1].dimshuffle('x', 0, 'x', 'x'))
lds3 = pool_2d(lconv3, (2, 2))
return X, lconv1, lds1, lconv2, lds2, lconv3, lds3
def pooling(self, inputs, pool_size, ignore_border, stride, pad, mode):
if pool_size == [1, 1]:
return inputs
if mode == "avg":
mode = "average_exc_pad"
if mode == "fmp":
height = inputs.shape[2]
width = inputs.shape[3]
batch = inputs.shape[0]
X = inputs.dimshuffle(2, 3, 0, 1) # (row, col, batches, filters)
sizes = T.zeros((batch, 2))
sizes = T.set_subtensor(sizes[:, 0], height)
sizes = T.set_subtensor(sizes[:, 1], width)
pooled_out, _ = fmp(X, sizes, pool_size[0])
return pooled_out.dimshuffle(2, 3, 0, 1)
pool_out = pool.pool_2d(
input=inputs,
ds=pool_size,
ignore_border=ignore_border,
st=stride,
padding=pad,
mode=mode
)
pool_out.name = "pool_out_"+self.name
return pool_out
def get_output_for(self, input, **kwargs):
if self.pad == 'strictsamex':
assert(self.stride[0] == 1)
kk = self.pool_size[0]
ll = int(np.ceil(kk/2.))
# rr = kk-ll
# pad = (ll, 0)
pad = [(ll, 0)]
length = input.shape[2]
self.ignore_border = True
input = padding.pad(input, pad, batch_ndim=2)
pad = (0, 0)
else:
pad = self.pad
pooled = pool.pool_2d(input,
ds=self.pool_size,
st=self.stride,
ignore_border=self.ignore_border,
padding=pad,
mode=self.mode,
)
if self.pad == 'strictsamex':
pooled = pooled[:, :, :length or None, :]
return pooled
def CNN(x,c_l1,c_l2,f_l1,f_l2,PP,ims):
print ims
#-------
#conv3D get rid of dependency of the number of input image channel
b=numpy.zeros(c_l1.get_value().shape[0])
conv1=tensor.nnet.relu(conv3D(x.dimshuffle(0,2,3,1,'x'),c_l1.dimshuffle(0,2,3,1,'x'),b,d=(1,1,1))) # shuffle dimensions
conv1=tensor.sum(conv1,axis=3) #add the dimension of channels
conv1=conv1.dimshuffle(0,3,1,2) #shuffle back to same dimension as conv2D
#---------
#conv1=tensor.nnet.relu(conv2d(x,c_l1)) #default stride=1 --subsample=(1,1)
conv1_shp=get_conv_output_shape(ims,c_l1.get_value().shape,border_mode='valid',subsample=(1,1))
print conv1_shp
#pp=tensor.reshape(conv1,conv1_shp[:2]+(conv1_shp[2]*conv1_shp[3],))
#print pp
pool1=pool_2d(conv1,(2,2),st=(2,2),ignore_border=True) #default maxpool
pool1_shp=get_pool_output_shape(conv1_shp,pool_size=(2,2),st=(2,2),ignore_border=True)
print pool1_shp
conv2=tensor.nnet.relu(conv2d(pool1,c_l2))
conv2_shp=get_conv_output_shape(pool1_shp,c_l2.get_value().shape,border_mode='valid',subsample=(1,1))
print conv2_shp
#pool2=pool_2d(conv2,(2,2),st=(2,2),ignore_border=True)
pool2=spp(conv2,conv2_shp,PP,'max')
fpool2=tensor.flatten(pool2,outdim=2)
full1=tensor.nnet.relu(tensor.dot(fpool2,f_l1))
pyx=tensor.nnet.softmax(tensor.dot(full1,f_l2))
return c_l1, c_l2, f_l1, f_l2, pyx
def test_mkl_elemwise_sum_forward():
x = tensor.ftensor4('x')
y1 = tensor.nnet.relu(x)
y2 = pool.pool_2d(y1, (1, 1), ignore_border=False, mode='max')
z = tensor.nnet.relu(y1)
out = y2 + z
f = theano.function(inputs=[x], outputs=out, mode=mode_with_mkl)
topo = f.maker.fgraph.toposort()
# inputs = f.maker.fgraph.inputs
# outputs = f.maker.fgraph.outputs
assert len(topo) == 6
assert isinstance(topo[4].op, mkl_elemwise.ElemwiseSum)
assert isinstance(topo[5].op, basic_ops.I2U)
imval = numpy.random.rand(4, 2, 4, 4).astype(numpy.float32)
new_out = f(imval)
forig = theano.function(inputs=[x], outputs=out, mode=mode_without_mkl)
topo_orig = forig.maker.fgraph.toposort()
assert len(topo) != len(topo_orig)
ref_out = forig(imval)
assert numpy.allclose(new_out, ref_out)
print('test_mkl_elemwise_sum_forward() pass..')
def model(X, params, pDropConv, pDropHidden):
lnum = 0 # conv: (32, 32) pool: (16, 16)
layer = nin(X, params[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (16, 16) pool: (8, 8)
layer = nin(layer, params[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (8, 8) pool: (4, 4)
layer = nin(layer, params[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1
layer = gap(layer, params[lnum])
return softmax(layer) # 如果使用nnet中的softmax训练产生NAN
def test_convolution(self):
"""
input: a 4D tensor corresponding to a mini-batch of input images. The shape of the tensor is as follows:
[mini-batch size, number of input feature maps, image height, image width].
"""
self.input = T.tensor4(name='input')
#Weights
W_shape = (self.numbers_of_feature_maps[1],self.numbers_of_feature_maps[0],self.filter_shape[0],self.filter_shape[1])
w_bound = np.sqrt(self.numbers_of_feature_maps[0]*self.filter_shape[0]*self.filter_shape[1])
self.W = theano.shared( np.asarray(np.random.uniform(-1.0/w_bound,1,0/w_bound,W_shape),dtype=self.input.dtype), name = 'W' )
#Bias
bias_shape = (self.numbers_of_feature_maps[1],)
self.bias = theano.shared(np.asarray(
np.random.uniform(-.5,.5, size=bias_shape),
dtype=input.dtype), name ='b')
#Colvolution
self.convolution = conv.conv2d(self.input,self.W)
self.max_pooling = pool.pool_2d(
input=self.convolution,
ds=self.pooling_size,
ignore_border=True
)
output = T.tanh(self.convolution + self.bias.dimshuffle('x', 0, 'x', 'x'))
f = theano.function([input], output)
def pool2d(x, pool_size, strides=(1, 1), border_mode='valid',
dim_ordering='th', pool_mode='max'):
# ====== dim ordering ====== #
if dim_ordering not in {'th', 'tf'}:
raise Exception('Unknown dim_ordering ' + str(dim_ordering))
if dim_ordering == 'tf':
x = x.dimshuffle((0, 3, 1, 2))
# ====== border mode ====== #
if border_mode == 'same':
w_pad = pool_size[0] - 2 if pool_size[0] % 2 == 1 else pool_size[0] - 1
h_pad = pool_size[1] - 2 if pool_size[1] % 2 == 1 else pool_size[1] - 1
padding = (w_pad, h_pad)
elif border_mode == 'valid':
padding = (0, 0)
elif isinstance(border_mode, (tuple, list)):
padding = tuple(border_mode)
else:
raise Exception('Invalid border mode: ' + str(border_mode))
# ====== pooling ====== #
if _on_gpu() and dnn.dnn_available():
pool_out = dnn.dnn_pool(x, pool_size,
stride=strides,
mode=pool_mode,
pad=padding)
else: # CPU veresion support by theano
pool_out = pool.pool_2d(x, ds=pool_size, st=strides,
ignore_border=True,
padding=padding,
mode=pool_mode)
if dim_ordering == 'tf':
pool_out = pool_out.dimshuffle((0, 2, 3, 1))
return pool_out
def test_dnn_tag():
"""
Test that if cudnn isn't avail we crash and that if it is avail, we use it.
"""
x = T.ftensor4()
old = theano.config.on_opt_error
theano.config.on_opt_error = "raise"
sio = StringIO()
handler = logging.StreamHandler(sio)
logging.getLogger('theano.compile.tests.test_dnn').addHandler(handler)
# Silence original handler when intentionnally generating warning messages
logging.getLogger('theano').removeHandler(theano.logging_default_handler)
raised = False
try:
f = theano.function(
[x],
pool_2d(x, ds=(2, 2), ignore_border=True),
mode=mode_with_gpu.including("cudnn"))
except (AssertionError, RuntimeError):
assert not dnn.dnn_available(test_ctx_name)
raised = True
finally:
theano.config.on_opt_error = old
logging.getLogger(
'theano.compile.tests.test_dnn').removeHandler(handler)
logging.getLogger('theano').addHandler(theano.logging_default_handler)
if not raised:
assert dnn.dnn_available(test_ctx_name)
assert any([isinstance(n.op, dnn.GpuDnnPool)
for n in f.maker.fgraph.toposort()])
def __init__(self, input, filter_shape, image_shape, poolsize=(2,2)):
# batch_size * filter_height * filter_width
fan_in = np.prod(filter_shape[1:])
# feature_maps * filter_height * filter_width / pooling_size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) / np.prod(poolsize))
W_bounds = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(np.asarray(rng.uniform(
low = -W_bounds, high = W_bounds, size=filter_shape),
dtype=theano.config.floatX), borrow=True)
# one bias per feature map
b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv2d( input = input, filters = self.W, filter_shape = filter_shape, input_shape = image_shape)
# downsample using max pooling
pooled_out = pool.pool_2d( input = conv_out, ds = poolsize, ignore_border = True)
# add bias
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
self.input = input
def test_DownsampleFactorMax(self):
rng = numpy.random.RandomState(utt.fetch_seed())
# generate random images
maxpoolshps = ((1, 1), (2, 2), (3, 3), (2, 3))
imval = rng.rand(4, 2, 16, 16)
images = tensor.dtensor4()
for maxpoolshp, ignore_border, mode in product(
maxpoolshps, [True, False], ["max", "sum", "average_inc_pad", "average_exc_pad"]
):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
# Pure Numpy computation
numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp, ignore_border, mode=mode)
output = pool_2d(images, maxpoolshp, ignore_border, mode=mode)
f = function([images], [output])
output_val = f(imval)
utt.assert_allclose(output_val, numpy_output_val)
# Pool op
maxpool_op = Pool(maxpoolshp, ignore_border=ignore_border, mode=mode)(images)
output_shape = Pool.out_shape(imval.shape, maxpoolshp, ignore_border=ignore_border)
utt.assert_allclose(numpy.asarray(output_shape), numpy_output_val.shape)
f = function([images], maxpool_op)
output_val = f(imval)
utt.assert_allclose(output_val, numpy_output_val)
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(1, 1), stride=(1,1)):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height,filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows,#cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(
rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv.conv2d(input=input, filters=self.W,
filter_shape=filter_shape, image_shape=image_shape)
# downsample each feature map individually, using maxpooling
pooled_out = pool.pool_2d(input=conv_out,
ds=poolsize, ignore_border=True, st=stride)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1,n_filters,1,1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# No. of inputs to a hidden unit =
# input_feature_maps * filter_height * filter_width
fan_in = numpy.prod(filter_shape[1:])
# Each unit in the lower layer recieves gradients
# from num_output_feature_maps * filter_height * filter_width
# / pooling_size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(poolsize))
W_values = xavier_init(rng, fan_in, fan_out, T.tanh, filter_shape)
self.W = theano.shared(W_values, borrow=True)
b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(b_values, borrow=True)
# Convolution operation (input feature maps with filters)
conv_out = conv2d(
input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape
)
# Apply max-pooling (downsample)
# Notice that instead of padding 0s, the border is ignored
pooled_out = pool_2d(
input=conv_out,
ds=poolsize,
ignore_border=True
)
# Dimshuffle bias vector to allow one bias term per filter
# The rest will be handled via broadcasting
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
self.input = input
def encode(self, utt_j, uttcut_j):
# transform word embedding to hidden size
emb_j = T.tanh( self.Wemb[utt_j[:uttcut_j],:] )
# 1st convolution
wh1_j = self.convolution(emb_j,self.Wcv1)
if self.pool[0]: # pooling
wh1_j = pool.max_pool(input=wh1_j,ds=(3,1),ignore_border=False)
wh1_j = T.tanh(wh1_j)
# 2nd convolution
wh2_j = self.convolution(wh1_j, self.Wcv2)
if self.pool[1]: # pooling
wh2_j = pool.max_pool(input=wh2_j,ds=(3,1),ignore_border=False)
wh2_j = T.tanh(wh2_j)
if self.level>=3:
# 3nd convolution
wh3_j = self.convolution(wh2_j, self.Wcv3)
if self.pool[2]:
wh3_j = pool.pool_2d(input=wh3_j,ds=(3,1),
ignore_border=False)
# average pooling
wh3_j = T.tanh(T.sum(wh3_j,axis=0))
else: # level < 3
wh3_j = None
if self.pool==(True,True,True):
return _, wh3_j
else:
return T.concatenate([wh1_j,wh2_j],axis=1), wh3_j
def test_max_pool_2d_2D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1), (3, 2))
imval = rng.rand(4, 5)
images = tensor.dmatrix()
for maxpoolshp, ignore_border, mode in product(maxpoolshps,
[True, False],
['max', 'sum',
'average_inc_pad',
'average_exc_pad']):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp,
ignore_border,
mode=mode)
output = pool_2d(images, maxpoolshp, ignore_border,
mode=mode)
output_val = function([images], output)(imval)
utt.assert_allclose(output_val, numpy_output_val)
def mp(input):
return pool_2d(input, maxpoolshp, ignore_border,
mode=mode)
utt.verify_grad(mp, [imval], rng=rng)
def run_test(direction='forward'):
print ('=' * 60)
print ('generate relu_pool graph before and after opt for %s pass' % direction)
x = T.ftensor4('x')
maxpoolshp = (2, 2)
ignore_border = False
mode = 'max'
imval = np.random.rand(4, 2, 16, 16).astype(np.float32)
reluOut = T.nnet.relu(x)
poolOut = pool.pool_2d(reluOut, maxpoolshp, ignore_border, mode=mode)
if direction == 'forward':
theano.printing.pydotprint(poolOut, outfile="relu_pool_before_opt.png", var_with_name_simple=True)
f = theano.function(inputs=[x], outputs=[poolOut])
theano.printing.pydotprint(f, outfile="relu_pool_after_opt.png", var_with_name_simple=True)
f(imval)
elif direction == 'backward':
poolSum = T.sum(poolOut)
poolBackward = T.grad(poolSum, [x])
theano.printing.pydotprint(poolBackward, outfile="relu_poolBackward_before_opt.png", var_with_name_simple=True)
f = theano.function(inputs=[x], outputs=poolBackward)
theano.printing.pydotprint(f, outfile="relu_poolBackward_after_opt.png", var_with_name_simple=True)
f(imval)
else:
print ("Invalid direction, only forward or backward allowed!")
def __init__(self, rng, input, layer_shape, input_shape, pool_size = (2,2)):
'''
:param rng: random number generator
:param input: 4D tensor with shape of input_shape
:param layer_shape: 4D matrix, out_put_num * input_num * kernel_rows * kernel_cols
:param input_shape: 4D matrix, batch_size * input_num * image_rows * image_cols
:param pool_size: pool_size
:return: Nothing
'''
assert input_shape[1] == layer_shape[1]
self.input = input
fan_in = np.prod(layer_shape[1:])
fan_out = (layer_shape[0] * np.prod(layer_shape[2:])) // np.prod(pool_size)
W_bound = np.sqrt(6.0 / (fan_out + fan_in))
self.W = theano.shared(np.array(rng.uniform(low = - W_bound, high= W_bound, size = layer_shape), dtype = theano.config.floatX),
borrow=True)
self.b = theano.shared(np.zeros(shape = (layer_shape[0], ), dtype = theano.config.floatX), borrow = True)
convolution_out = conv2d(input, self.W, filter_shape = layer_shape, input_shape = input_shape) #what will happen if I delete the last two parameters
pool_out = downsample.pool_2d(convolution_out, pool_size, ignore_border = True)
self.output = T.tanh(pool_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
def layer(no_filters, inp_layers, filter_x, filter_y, pool_x, pool_y, inp_img, layer_id):
print '----------------------------------------------------------------------------------------'
input = T.tensor4(name='input')
w_shp = (no_filters, inp_layers, filter_x, filter_y) #no_filters * inputmap_count * filter_x * filter_y
w_bound = numpy.sqrt(inp_layers * filter_x * filter_y) #total_no_weights for every output variable
W = theano.shared(numpy.asarray( rng.uniform(low = -1.0/w_bound, high=1.0/w_bound, size = w_shp), dtype=input.dtype), name='W')
#print 'Type of W:', type(W); print W.shape.eval(); print W[0].eval() #takes time to print
b_shp = (no_filters,)
b = theano.shared(numpy.asarray(rng.uniform(low = -.5, high=.5, size = b_shp), dtype=input.dtype), name='b')
conv_out = conv2d(input, W) #this is called as a symbolic expression
output = T.nnet.sigmoid(conv_out + b.dimshuffle('x',0,'x','x'))
f = theano.function([input], output)
print 'Layer', layer_id,': Applying conv2d: ', inp_img.shape
filtered_img = f(inp_img)
save_images(layer_id, no_filters, filtered_img, 'conv2d')
maxpool_shape = (pool_x, pool_y)
pool_out = pool.pool_2d(input, maxpool_shape, ignore_border=True)
fpool = theano.function([input], pool_out)
print 'Layer', layer_id,': Pooling: ',filtered_img.shape
pooled_img = fpool(filtered_img)
save_images(layer_id, no_filters, pooled_img, 'pool')
return pooled_img
def load_data(reduce_dim,dir_name,up_to=None,max_pool=True):
"""
Load image data from a folder into a single theano shared variable.
The directory should only contain the images to be used.
args:
-reduce_dim: a tuple or list containing the dimensionality of the crop to make.
if max_pool is false, this gets used for cropping.
if max_pool is true, this gets used for max pooling
-dir_name: the directory from which to extract data
"""
if max_pool:
x = T.matrix('x')
fpool = theano.function(inputs=[x],outputs= pool.pool_2d(x,reduce_dim,ignore_border=True))
csv_file = "img{}_{}-{}maxpool.csv"
else:
csv_file = "img{}_{}-{}.csv"
fpool = None
imgs = []
os.chdir(dir_name)
fig = plt.figure()
ax = fig.add_subplot(111)
file_list = os.listdir(dir_name)
file_list = [name for name in file_list if ".png" in name] #only want png files
print(file_list[:10])
if (up_to > len(file_list)):
print("Can't iterate through more files than actually exist!")
return None
elif up_to == None:
up_to = len(file_list)
dim = ""
t0 = time.time()
for i, img_name in enumerate(file_list):
t1 = time.time()
if i == up_to:
break
im = imread(img_name)
r = im[:,:,0]
b = im[:,:,1]
g = im[:,:,2]
grey = 0.2126*r + 0.7152*g + 0.0722*b
if not max_pool:
w = grey.shape[0] #this is inefficient
h = grey.shape[1]
w_c = reduce_dim[0] #width crop
h_c = reduce_dim[1] #height crop
dim = (w_c, h_c)
crop = grey[int(w/2)-int(w_c/2):int(w/2)+int(w_c/2),int(h/2)-int(h_c/2):int(h/2)+int(h_c/2)]
if max_pool:
crop = fpool(grey)
dim = crop.shape
# ax.imshow(crop,cmap='gray')
# plt.show()
crop /= np.amax(crop)
imgs.append(crop.flatten())
print("Time for this iteration: {:.2f}, only {} more to go!".format(time.time()-t1, up_to-i))
csv_file = csv_file.format(str(up_to), str(dim[0]), str(dim[1]))
np.savetxt(csv_file, np.asarray(imgs), delimiter=",")
print("Total time iterating through images and saving: {:.2f}".format(time.time()-t0))
def __init__(self, input, poolsize=(2,2)): pooled_out = pool_2d( input=input, ds=poolsize, ignore_border=True, mode="max" ) self.output = pooled_out
def test_max_pool_2d_3D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = [(1, 2)]
imval = rng.rand(2, 3, 4)
images = tensor.dtensor3()
for maxpoolshp, ignore_border, mode in product(
maxpoolshps, [True, False],
['max', 'sum', 'average_inc_pad', 'average_exc_pad']):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp,
ignore_border, mode)
output = pool_2d(images, maxpoolshp, ignore_border, mode=mode)
output_val = function([images], output)(imval)
utt.assert_allclose(output_val, numpy_output_val)
def convpool(X, W, b, poolsize=(2, 2)):
conv_out = conv2d(input=X, filters=W)
# downsample each feature map individually, using maxpooling
pooled_out = pool.pool_2d(
input=conv_out,
ws=poolsize,
ignore_border=True
)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
# return T.tanh(pooled_out + b.dimshuffle('x', 0, 'x', 'x'))
return relu(pooled_out + b.dimshuffle('x', 0, 'x', 'x'))
def __init__(self,
net,
input):
# Save config information to its layer
self.Ws = net.LayerOpts['pool_filter_size']
self.IgnoreBorder = net.LayerOpts['pool_ignore_border']
self.Stride = net.LayerOpts['pool_stride']
self.Padding = net.LayerOpts['pool_padding']
self.Mode = net.LayerOpts['pool_mode']
self.Output = pool_2d(input = input,
ws = self.Ws,
ignore_border = self.IgnoreBorder,
stride = self.Stride,
pad = self.Padding,
mode = self.Mode)
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape)
# pool each feature map individually, using maxpooling
pooled_out = pool.pool_2d(input=conv_out,
ws=poolsize,
ignore_border=True)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
# keep track of model input
self.input = input
def __init__(self, rstream, index, x,
params, useRglrz, bnPhase,
poolShape, inFilters, outFilters, stride, ignore_border = False,
b=None, a=None, normParam=None, rglrzParam=None):
'''
Averaging layer + BN + noise
'''
# noise
self.paramsT2 = []
if 'addNoise' in params.rglrz and params.convLayers[index].noise:
if rglrzParam is None:
self.rglrzParam = {}
tempValue = params.rglrzInitial['addNoise'][index]
tempParam = np.asarray(tempValue, dtype=theano.config.floatX)
noizParam = theano.shared(value=tempParam, name='%s_%d' % ('addNoise', index), borrow=True)
self.rglrzParam['addNoise']=noizParam
if params.useT2 and 'addNoise' in params.rglrzTrain:
self.paramsT2 = [noizParam]
#self.output = noiseup(self.output, splitPoint, noizParam, params.noiseT1, params, index, rstream)
x = noiseup(x, useRglrz, noizParam, params.noiseT1, params, index, rstream)
# averaging
if cudasConv:
self.output = cudnn.dnn_pool(x, poolShape, stride = stride, mode = 'average_exc_pad')#, ignore_border = ignore_border)
else:
self.output = pool.pool_2d(x, ds = poolShape, st = stride, ignore_border = ignore_border, mode = 'average_exc_pad')
# if batch normalization
if params.batchNorm and params.convLayers[index].bn:
_, b, a = t1_shared(params=params, rng=0, index=index, nIn=0, nOut=0,
outFilters=outFilters, filterShape=0, defineW=0)
self.b = b; self.a = a
self.paramsT1 = [b]
if normParam is None:
normParam, paramsBN = bn_shared(params, outFilters, index)
self.normParam = normParam
self.paramsBN = paramsBN
self.output, updateBN = bn_layer(self.output, self.a, self.b, self.normParam, params, bnPhase)
self.updateBN = updateBN
# flattening and softmax
self.output = T.flatten(self.output, outdim = 2)
if params.convLayers[index].type == 'average+softmax':
self.output = activation(self.output, 'softmax')
def __call__(self, X):
# X: (batch_size, max_sent_len, word_embed_dim)
# valid: (batch_size, nb_filters, max_sent_len - filter_window_size + 1, 1)
# full: (batch_size, nb_filters, max_sent_len + filter_window_size - 1, 1)
conv_output = conv.conv2d(X.reshape((X.shape[0], 1, X.shape[1], X.shape[2])),
filters=self.W,
filter_shape=self.W.shape.eval(),
border_mode=self.border_mode)
output = self.activation(conv_output + self.b.dimshuffle(('x', 0, 'x', 'x')))
# (batch_size, nb_filters, 1, 1)
output = pool.pool_2d(output, ds=self.ds, ignore_border=True, mode='max')
# (batch_size, nb_filters)
output = output.flatten(2)
return output
def cnn_layer(tparams, proj, options):
#proj = proj.dimshuffle(1,'x',0,2) #(batchsize,1,max_len,dim_proj)
proj = proj.dimshuffle(
1, 3, 0, 2
) # (maxlen,n_sample(batchsize), dim_proj, num_chn) -> (batchsize,num_chn,max_len,dim_proj)
#image_shape = proj.shape
filter_shapes = options['filter_shapes']
image_shape = options['image_shape']
pool_sizes = options['pool_sizes']
image_shape = (None, image_shape[1], image_shape[2], image_shape[3])
conv_outs = []
for i in range(len(filter_shapes)):
filter_shape = filter_shapes[i]
pool_size = pool_sizes[i]
#img_h = image_shape[2]
filter_h = filter_shape[2]
#img_w = image_shape[3]
#filter_w = filter_shape[3]
#poolsize = (img_h-filter_h+1,img_w-filter_w+1)
conv_out = conv.conv2d(input=proj,
filters=tparams['cnn_f' + str(filter_h)],
filter_shape=filter_shape,
image_shape=image_shape)
conv_out_relu = ReLU(
conv_out +
tparams['cnn_b' + str(filter_h)].dimshuffle('x', 0, 'x', 'x'))
if options['pool_type'] == 'max':
conv_out_pool = pool.pool_2d(input=conv_out_relu,
ds=pool_size,
ignore_border=True,
mode='max')
# conv_out_pool = pool.pool_2d(input=conv_out_relu,ds=pool_size,ignore_border=True,mode='max')
elif options['pool_type'] == 'avg':
conv_out_pool = conv_out_relu.flatten(3)
conv_out_pool = tensor.mean(conv_out_pool, axis=2)
else:
sys.exit()
conv_outs.append(conv_out_pool.flatten(2))
proj = tensor.concatenate(conv_outs, 1)
return proj
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
"""
rng:随机数生成器
input:输入的批量测试数据
filter_shape:list of lenght 4，(number of filters, num input feature maps, filter height, filter width); 这里要注意 的number of filters表示的是这个层有多少个输出特征图，也就是说每个输入特征图会有多少个卷积核；而不是这个层有的全部的卷积核数
image_shape: list of length 4, (batch size[表示批量的数据，也就是一次计算使用的图片数], num input feature maps, image height, image width)
poolsize:池化的核大小
"""
print image_shape[1], filter_shape[1]
assert image_shape[1] == filter_shape[1]
self.input = input
fan_in = numpy.prod(
filter_shape[1:]
) #本层每个输出神经元与多少个输入神经元相连 num input feature maps* filter height * filter width
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:])) / numpy.prod(
poolsize
) #每个输入神经元可以从几个输出神经元得到误差加传"num output feature maps * filter height * filter width / pooling size"
W_bound = numpy.sqrt(6.0 / (fan_in + fan_out))
self.W = theano.shared(
numpy.asarray(
rng.uniform(
low=-W_bound, high=W_bound, size=filter_shape
), #(number of filters, num input feature maps, filter height, filter width)；每个单体是一个hieght * width大小的矩阵，表示一个核，每number input feature maps 个单体组成一个个体，表示所有输入特征图的一组核，而一共有number of filtes组；其实这个也可以看作是每列都代表了一个输出特征图的一组核
dtype=theano.config.floatX),
borrow=True)
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
#卷积输出
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape)
#池化
pooled_out = pool.pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True)
#计算的结果还要加上偏置，但是我们的输出pool_out是一个(batch size, output feature map, height, width)这样一个四维的向量，而b是一个一维的向量，所以要将b进行扩展成(1, output feature map, 1, 1),再利用numpy的broadcasted进行相加；则每个批量的用例中的每个特征图的值都会加上这个特征图对应的偏置了
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
self.input = input
def __init__(self, input, ds, mode='max'):
"""
Pooling layer class
:type input: theano.tensor.dmatrix
:param input: a symbolic tensor
:type ds: tuple of length 2
:param ds: a factor by which to downscale
:type mode: string
:param mode: pooling mode {'max', 'sum', 'average_inc_pad', 'average_exc_pad'}
"""
self.input = input
self.output = pool.pool_2d(input=input,
ds=ds,
ignore_border=True,
mode=mode)
self.params = []
def pooling(input_pool, size, ignore_border=False):
# Function to perform Max-Pooling on each feature map
# Inputs:
# input_pool - feature maps obtained as output from convolution layer.
# size - specification of downsampling (pooling) factor.
# tuple format: (# of rows, # of columns)
# Outputs:
# pool_out - pooled output.
# dimensions: (# of channels, conv_output_height/#rows,
# conv_output_width/#rows)
pool_out = pool.pool_2d(input=input_pool,
ws=size,
ignore_border=ignore_border)
return pool_out
def __init__(self,
rng,
input,
filter_shape,
image_shape,
poolsize=(2, 2),
activation=T.tanh):
assert image_shape[1] == filter_shape[1]
self.input = input
fan_in = numpy.prod(filter_shape[1:])
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(poolsize))
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
self.b = theano.shared(numpy.zeros((filter_shape[0], ),
dtype=theano.config.floatX),
borrow=True)
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape)
conv_out = conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
if not activation is None:
conv_out = activation(conv_out)
pool_out = pool.pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True)
# self.output = T.tanh(pool_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.output = pool_out
self.params = [self.W, self.b]
self.input = input
def pool2d(self, x, pool_size, strides=(1, 1), border_mode='valid',
pool_mode='max'):
if border_mode == 'same':
# TODO: add implementation for border_mode="same"
raise Exception('border_mode="same" not supported with Theano.')
elif border_mode == 'valid':
ignore_border = False
padding = (0, 0)
else:
raise Exception('Invalid border mode: ' + str(border_mode))
pool_out = pool.pool_2d(x, ds=pool_size,
ignore_border=ignore_border,
padding=padding,
mode=pool_mode
)
return pool_out
def pool_fn(pool_size,
ignore_border=False,
stride=None,
pad=(0, 0),
mode='max'):
if mode == 'avg': mode = 'average_exc_pad'
if isinstance(pool_size, (int, float)):
pool_size = (int(pool_size), int(pool_size))
xt = T.tensor4()
poolx = pool_2d(xt,
pool_size,
ignore_border=ignore_border,
st=stride,
padding=pad,
mode=mode)
pool = theano.function([xt], poolx, allow_input_downcast=True)
return pool
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = filter_shape[0] * numpy.prod(
filter_shape[2:]) / numpy.prod(poolsize)
# initialize weights with random weights
W_bound = numpy.sqrt(6.0 / (fan_in + fan_out))
self.W = theano.shared(
numpy.asarray(
rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX,
),
borrow=True,
)
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv.conv2d(
input=input,
filters=self.W,
filter_shape=filter_shape,
image_shape=image_shape,
)
# downsample each feature map individually, using maxpooling
pooled_out = downsample.pool_2d(input=conv_out,
ws=poolsize,
ignore_border=True)
self.output = T.maximum(
0.0, pooled_out + self.b.dimshuffle("x", 0, "x", "x"))
# store parameters of this layer
self.params = [self.W, self.b]
def pool2d(x, pool_size=(2, 2), strides=None, border_mode=(0, 0),
ignore_border=True, mode='max'):
"""
Parameters
----------
x : N-D theano tensor of input images
Input images. Max pooling will be done over the 2 last dimensions.
pool_size : tuple of length 2 or theano vector of ints of size 2.
Factor by which to downscale (vertical ws, horizontal ws).
(2, 2) will halve the image in each dimension.
strides : tuple of two ints or theano vector of ints of size 2.
Stride size, which is the number of shifts over rows/cols to get the
next pool region. If stride is None, it is considered equal to ws
(no overlap on pooling regions).
border_mode : tuple of two ints or theano vector of ints of size 2.
(pad_h, pad_w), pad zeros to extend beyond four borders of the
images, pad_h is the size of the top and bottom margins, and
pad_w is the size of the left and right margins.
ignore_border : bool (default None, will print a warning and set to False)
When True, (5,5) input with ws=(2,2) will generate a (2,2) output.
(3,3) otherwise.
mode : {'max', 'avg'}
Operation executed on each window. `max` or `average`
Note
----
This pooling algorithm has non-deterministic behaviour on cuDNN
"""
# ====== convert to theano formated shapes ====== #
input_shape = get_shape(x)
# pool_size
x, pool_size, strides, border_mode, mode = __validate_pool_stride_border(
x, pool_size, strides, border_mode, mode, ndim=2)
x = __img_theano_format(x)
# ====== On GPU: use CuDNN ====== #
pool_out = pool.pool_2d(x, ws=pool_size, stride=strides,
ignore_border=ignore_border,
pad=(0, 0) if isinstance(border_mode, str) else border_mode,
mode=mode)
# ====== Estimate output shape ====== #
pool_out = __img_tensorflow_format(pool_out)
output_shape = get_pool_output_shape(input_shape, pool_size,
ignore_border=ignore_border, strides=strides, pad=border_mode)
add_shape(pool_out, tuple(output_shape))
return pool_out
def apply(self, input_data):
conv_out = conv2d(input=input_data,
filters=self.W,
input_shape=self.input_shape,
filter_shape=self.filter_shape,
border_mode='valid')
# conv_row_len = input_shape[2] - filter_shape[2] + 1
pool_out = pool.pool_2d(input=conv_out,
ds=self.pool_size,
ignore_border=True,
mode=self.pooling_op)
lin_output = pool_out + self.b.dimshuffle(
'x', 0, 'x', 'x') # D * Filter * out_h * out_w
if self.activation is None:
return lin_output
else:
tensor_output = self.activation(lin_output)
return tensor_output
def __init__(self,x,image_shape,filter_shape,poolsize=(2,2)):
#randomly initial W and b
print image_shape[1]
print filter_shape[1]
#assert image_shape[1]==filter_shape[1] #channal shouble be equal
factor = np.prod(filter_shape[1:])
low=-np.sqrt(1./np.sqrt(factor))
high=np.sqrt(1./np.sqrt(factor))
random_w = np.random.uniform(low=low,high=high,
size=filter_shape)
#print random_w
self.W = shared(np.asarray(random_w,dtype=theano.config.floatX))
self.b = shared(np.zeros(filter_shape[0]
,dtype=theano.config.floatX))
self.params=[self.W,self.b]
conv_out=T.nnet.conv2d(x,self.W)
pool_out=pool_2d(conv_out,poolsize,ignore_border=True)
self.output=T.nnet.sigmoid(pool_out+self.b.dimshuffle('x',0,'x','x'))
def set_inpt(self, inpt, inpt_dropout, mini_batch_size=None):
if not mini_batch_size:
self.inpt = inpt.reshape(self.image_shape)
conv_out = conv2d(
input=self.inpt, filters=self.w,
filter_shape=self.filter_shape, image_shape=self.image_shape)
else:
new_shape = list(self.image_shape)
new_shape[0] = mini_batch_size
self.inpt = inpt.reshape(new_shape)
conv_out = conv2d(
input=self.inpt, filters=self.w,
filter_shape=self.filter_shape, input_shape=new_shape)
pooled_out = pool.pool_2d(
input=conv_out, ws=self.poolsize, ignore_border=True)
self.output = self.activation_fn(
pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.output_dropout = self.output # no dropout in the convolutional
def __init__(self, input, pool_size, padding=(0, 0), stride=None, mode='max', ignore_border=True, activation=None):
assert input.ndim >= 3, "Incompatibility: ndim-{} of input-{} is not >= 3".format(input.ndim, input)
super(Pool2D, self).__init__(input)
self.stride = pool_size if stride is None else stride
self.params = tuple()
self.padding = padding
self.shape = Pool2D.out_shape(input.shape, pool_size, self.stride, padding, ignore_border)
self.ndim = len(self.shape)
self.activation = activation
out = pool.pool_2d(self.input.out,
pool_size,
padding=padding,
st=stride,
ignore_border=ignore_border,
mode=mode)
self.out = out if activation is None else activation(out)
def conv_crop_pool_op(X, sizes, output_sizes, W, b, n_in, n_maps,
filter_height, filter_width, filter_dilation, poolsize):
global printed_cudnn_warning
if theano.sandbox.cuda.dnn.dnn_available():
conv_op = CuDNNConvHWBCOpValidInstance
pool_op = PoolHWBCOp(poolsize)
conv_out = conv_op(X, W, b) if filter_height * filter_width > 0 else X
crop_out = CropToBatchImageSizeInstance(conv_out, sizes)
Y = pool_op(crop_out)
Y = CropToBatchImageSizeZeroInstance(Y, output_sizes)
return Y
else:
if not printed_cudnn_warning:
print >> log.v2, "warning, cudnn not available, using theano conv implementation"
printed_cudnn_warning = True
#note: this solution uses alot of dimshuffles and so also alot of memory
#I only have this so that I can still run on my laptop for testing
#it's not really useful for productive use and also not much tested
filter_shape = (n_maps, n_in, filter_height, filter_width)
X_shuffled = X.dimshuffle(2, 3, 0, 1)
conv_out = conv.conv2d(
input=X_shuffled,
border_mode="valid",
filters=W,
filter_shape=filter_shape, # filter_dilation=filter_dilation,
image_shape=(None, n_in, None,
None)) if filter_height * filter_width > 0 else X
crop_out = CropToBatchImageSizeInstance(
conv_out.dimshuffle(2, 3, 0, 1), sizes).dimshuffle(2, 3, 0, 1)
if poolsize == (1, 1):
Y = crop_out
else:
#pooling cannot handle width > 512 (only with cuDNN), so we swap the axes and swap them back afterwards
crop_out = crop_out.dimshuffle(0, 1, 3, 2)
pooled_out = pool.pool_2d(
input=crop_out,
#max_pool_2d wants the sizes in the other order
ds=poolsize[::-1],
ignore_border=True)
#unshuffle it
Y = pooled_out.dimshuffle(0, 1, 3, 2)
Y = Y.dimshuffle(2, 3, 0, 1)
Y += b
return Y
def test_mkl_pool_forward():
maxpoolshps = ((1, 1), (2, 2), (3, 3), (2, 3))
imval = numpy.random.rand(4, 2, 16, 16).astype(theano.config.floatX)
if theano.config.floatX == 'float32':
images = tensor.ftensor4()
else:
images = tensor.dtensor4()
ignore_border = False
mode = 'max'
poolOut = pool.pool_2d(images, maxpoolshps[0], ignore_border, mode=mode)
f = theano.function(inputs=[images], outputs=[poolOut], mode=mode_with_mkl)
topo = f.maker.fgraph.toposort()
inputs = f.maker.fgraph.inputs
outputs = f.maker.fgraph.outputs
assert len(inputs) == 1
assert len(outputs) == 1
assert len(topo) == 3
assert isinstance(topo[0].op, U2IPool)
assert isinstance(topo[1].op, mkl.mkl_pool.Pool)
assert isinstance(topo[2].op, I2U)
# U2IPool
assert len(topo[0].inputs) == 4
assert isinstance(topo[0].inputs[1], TensorConstant)
assert isinstance(topo[0].inputs[3], TensorConstant)
assert topo[0].inputs[1] == topo[0].inputs[2]
# pool
assert len(topo[1].inputs) == 4
assert isinstance(topo[1].inputs[1], TensorConstant)
assert isinstance(topo[1].inputs[3], TensorConstant)
assert topo[1].inputs[1] == topo[1].inputs[2]
assert topo[1].inputs[0].owner == topo[0]
# I2U
assert len(topo[2].inputs) == 1
assert topo[2].inputs[0].owner == topo[1]
# Output
assert outputs[0].owner == topo[2]
f1 = theano.function(inputs=[images, ], outputs=[poolOut, ], mode=mode_without_mkl)
assert (numpy.asarray(f(imval)) == f1(imval)).all()
print('test_mkl_pool_forward() pass..')
def test_mkl_elemwise_sum_backward():
x = tensor.ftensor4('x')
y1 = tensor.nnet.relu(x)
y2 = pool.pool_2d(y1, (1, 1), ignore_border=False, mode='max')
z = tensor.nnet.relu(y1)
out = y2 + z
reluSum = tensor.sum(out)
reluBackward = tensor.grad(reluSum, [x])
f = theano.function(inputs=[x], outputs=reluBackward, mode=mode_with_mkl)
topo = f.maker.fgraph.toposort()
inputs = f.maker.fgraph.inputs
outputs = f.maker.fgraph.outputs
# assert len(topo.op) == 18
assert isinstance(topo[4].op, mkl_elemwise.ElemwiseSum)
imval = numpy.random.rand(4, 2, 4, 4).astype(numpy.float32)
f(imval)
print('test_mkl_elemwise_sum_backward() pass..')
def set_inpt(self, inpt, inpt_dropout, mini_batch_size):
"""
Input setter
:param inpt: image input from a layer
:param inpt_dropout: input dropout
:param mini_batch_size: mini batch size
"""
self.inpt = inpt.reshape(self.image_shape)
conv_out = nnet.conv2d(self.inpt,
self.weights,
filter_shape=self.filter_shape,
input_shape=self.image_shape)
pooled_out = pool.pool_2d(conv_out,
ws=self.pool_size,
ignore_border=True)
self.output = self.activation_fn(
pooled_out + self.biases.dimshuffle('x', 0, 'x', 'x'))
# no dropout in the conv layers
self.output_dropout = self.output
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
assert image_shape[1] == filter_shape[1]
self.input = input
fan_in = numpy.prod(filter_shape[1:])
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# 卷积
conv_out = conv.conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
image_shape=image_shape)
# 子采样 for 0.8x
"""
pooled_out = downsample.max_pool_2d(
input=conv_out,
ds=poolsize,
ignore_border=True
)
"""
#子采样for 0.9x
pooled_out = pool.pool_2d(input=conv_out,
ws=poolsize,
ignore_border=True)
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
def get_output(self, inputs):
"""
Get outputs of encoder layer.
Return all of the hidden status.
"""
(self.sentence, self.mask) = inputs
conv_out = conv.conv2d(input=self.sentence, \
filters=self.tparams[self._p(self.prefix, 'W')], \
filter_shape=self.filter_shape, \
image_shape=self.image_shape)
conv_out = conv_out * self.mask.dimshuffle(0, 'x', 1, 'x')
# downsample each feature map individually, using maxpooling
pooled_out = downsample.pool_2d(input=conv_out, ds=self.pool_size, \
ignore_border=True, mode='max')
output = tensor.tanh(pooled_out + self.tparams[self._p(self.prefix, 'b')]\
.dimshuffle('x', 0, 'x', 'x'))
return output[:, :, 0, 0], conv_out
def __init__(self, inpts, filter_shape, image_shape, pool_size=(2, 2)):
self.image_shape = image_shape
self.inpts = inpts.reshape(self.image_shape)
self.W = theano.shared(np.random.randn(5, 5))
self.B = theano.shared(np.random.randn(12, ))
self.params = [self.W, self.B]
self.z = theano.dot(self.inpts, self.W) + self.B
self.pool_size = pool_size
self.filter_shape = filter_shape
conv_out = conv.conv2d(self.inpts,
image_shape=self.image_shape,
filters=self.W,
filter_shape=self.filter_shape)
pool_out = downsample.pool_2d(input=conv_out,
ds=self.pool_size,
ignore_border=True)
self.out = T.nnet.sigmoid(pool_out)
def __init__(self, input, filter_w, filter_h, filter_num, img_w, img_h,
input_feature, batch_size, poolsize, params_W, params_b):
self.input = input
self.W = params_W
self.b = params_b
conv_out = conv.conv2d(input=input,
filters=self.W,
filter_shape=(filter_num, input_feature,
filter_h, filter_w),
image_shape=(batch_size, input_feature, img_h,
img_w))
pooled_out = pool.pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True)
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
def test_max_pool_2d_6D(self):
rng = np.random.RandomState(utt.fetch_seed())
maxpoolshps = [(3, 2)]
imval = rng.rand(2, 1, 1, 1, 3, 4)
images = tensor.TensorType("float64", [False] * 6)()
for maxpoolshp, ignore_border, mode in product(
maxpoolshps,
[True, False],
["max", "sum", "average_inc_pad", "average_exc_pad"],
):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(imval,
maxpoolshp,
ignore_border,
mode=mode)
output = pool_2d(images, maxpoolshp, ignore_border, mode=mode)
output_val = function([images], output)(imval)
utt.assert_allclose(output_val, numpy_output_val)
def __init__(self, input, rng, input_shape, filter_shape,
pool_size=(2, 2)):
self.input = input
self.rng = rng
fan_in = np.prod(filter_shape[1:])
w_value = np.array(self.rng.uniform(low=-3.0 / fan_in,
high=3.0 / fan_in,
size=filter_shape),
dtype=theano.config.floatX)
self.W = theano.shared(value=w_value, name='c_w', borrow=True)
b_value = np.zeros(filter_shape[0], dtype=theano.config.floatX)
self.b = theano.shared(value=b_value, name='c_b', borrow=True)
self.params = [self.W, self.b]
conv_out = conv.conv2d(self.input,
self.W,
image_shape=input_shape,
filter_shape=filter_shape)
max_pool_out = pool.pool_2d(conv_out, pool_size)
self.outputs = T.tanh(max_pool_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
def get_output_for(self, input, **kwargs):
# Power
input_powered = input**self.pnorm
# Average pool
avg_pooled = pool_2d(
input_powered,
ds=self.pool_size,
st=self.stride,
ignore_border=self.ignore_border,
padding=self.pad,
mode=self.mode,
)
# Scale with pooling kernel since we want the sum, not average
scaler = T.cast(np.prod(self.pool_size), dtype=theano.config.floatX)
scaled = avg_pooled * scaler
# De-power
depowered = scaled**(T.cast(1.0, dtype=theano.config.floatX) /
self.pnorm)
return depowered