def __init__(self, **kwargs):
"""Initializes the convolution layer. Strictly, this is a correlation
layer since the kernels are not reversed spatially as in a classical
convolution operation.
kwargs:
name: the name of the layer.
group: the number of groups that should be carried out for the
block group convolution. Note that the number of channels of
the incoming block should be divisible by the number of groups,
otherwise we will have an error produced.
num_kernels: the number of kernels PER GROUP. As a result, the
output would have (num_kernels * group) channels.
Also the layer should be provided all the appropriate parameters for
the underlying convolutional layer.
"""
base.Layer.__init__(self, **kwargs)
self._group = self.spec['group']
self._conv_args = dict(self.spec)
self._conv_args['name'] = self.spec['name'] + '_sub'
del self._conv_args['group']
self._bottom_sub = [base.Blob() for _ in range(self._group)]
self._top_sub = [base.Blob() for _ in range(self._group)]
self._conv_layers = None
self._blocksize = 0
self._num_kernels = self.spec['num_kernels']
# create the convolution layers
self._conv_layers = [
convolution.ConvolutionLayer(**self._conv_args)
for i in range(self._group)]
self._param = sum((layer.param() for layer in self._conv_layers), [])
return
def testLocalResponseNormalizeLayer(self):
np.random.seed(1701)
output_blob = base.Blob()
checker = gradcheck.GradChecker(1e-6)
shapes = [(1,10), (5,10)]
alphas = [1.0, 2.0]
betas = [0.75, 1.0]
for shape in shapes:
for alpha in alphas:
for beta in betas:
input_blob = base.Blob(shape, filler=fillers.RandFiller())
# odd size
layer = core_layers.LocalResponseNormalizeLayer(
name='normalize', k = 1., alpha=alpha, beta=beta, size=5)
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
layer = core_layers.LocalResponseNormalizeLayer(
name='normalize', k = 2., alpha=alpha, beta=beta, size=5)
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
# even size
layer = core_layers.LocalResponseNormalizeLayer(
name='normalize', k = 1., alpha=alpha, beta=beta, size=6)
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
def testGroupConvolutionGrad(self):
np.random.seed(1701)
output_blob = base.Blob()
checker = gradcheck.GradChecker(1e-3)
shapes = [(1, 5, 5, 4)]
num_kernels = 1
group = 2
params = [(3, 1, 'valid'), (3, 1, 'same'), (3, 1, 'full'),
(2, 1, 'valid'), (2, 1, 'full'), (3, 2, 'valid'),
(3, 2, 'same'), (3, 2, 'full')]
for shape in shapes:
for ksize, stride, mode in params:
print(ksize, stride, mode, shape)
input_blob = base.Blob(shape,
filler=fillers.GaussianRandFiller())
layer = core_layers.GroupConvolutionLayer(
name='gconv',
ksize=ksize,
stride=stride,
mode=mode,
num_kernels=num_kernels,
group=group,
filler=fillers.GaussianRandFiller())
result = checker.check(layer, [input_blob], [output_blob])
self.assertEqual(output_blob.data().shape[-1],
num_kernels * group)
print(result)
self.assertTrue(result[0])
# check if we will be able to produce an exception
input_blob = base.Blob((1, 5, 5, 3),
filler=fillers.GaussianRandFiller())
self.assertRaises(RuntimeError, checker.check, layer, [input_blob],
[output_blob])
def decaf_convolution(input_size, dtype, num_kernels, ksize, stride, mode,
iternum):
bottom = base.Blob((1, ) + input_size, dtype=dtype)
layer = convolution.ConvolutionLayer(name='conv',
num_kernels=num_kernels,
ksize=ksize,
stride=stride,
mode=mode)
top = base.Blob()
# run a forward pass first to initialize everything.
layer.forward([bottom], [top])
top.init_diff()
top.diff().flat = 1.
print '*****'
print 'input shape:', bottom.data().shape[1:]
start = time.time()
for i in range(iternum):
layer.forward([bottom], [top])
print 'forward runtime:', (time.time() - start) / iternum
print 'output shape:', top.data().shape[1:]
start = time.time()
for i in range(iternum):
layer.backward([bottom], [top], True)
print 'backward runtime:', (time.time() - start) / iternum
print '*****'
def setUp(self):
self.test_sizes = [(10,5), (1,1), (10,1), (1,5)]
self.test_output_sizes = [1, 5, 10]
self.test_blobs = [base.Blob(size, np.float32)
for size in self.test_sizes]
self.test_blobs += [base.Blob(size, np.float64)
for size in self.test_sizes]
def testReLUGrad(self):
np.random.seed(1701)
shapes = [(4, 3), (1, 10), (2, 5, 5, 1), (2, 5, 5, 3)]
output_blob = base.Blob()
layer = core_layers.ReLULayer(name='relu')
checker = gradcheck.GradChecker(1e-5)
for shape in shapes:
input_blob = base.Blob(shape, filler=fillers.GaussianRandFiller())
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
def testMeanNormalizeLayer(self):
np.random.seed(1701)
output_blob = base.Blob()
checker = gradcheck.GradChecker(1e-5)
shapes = [(1,5,5,1), (1,5,5,3), (5,5), (1,5)]
for shape in shapes:
input_blob = base.Blob(shape, filler=fillers.GaussianRandFiller())
layer = core_layers.MeanNormalizeLayer(
name='normalize')
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
def testIdentityLayer(self):
layer = identity.IdentityLayer(name='identity')
np.random.seed(1701)
filler = fillers.RandFiller()
bottom = base.Blob((100, 4), filler=filler)
top = base.Blob()
# run the dropout layer
layer.forward([bottom], [top])
# simulate a diff
fillers.RandFiller().fill(top.init_diff())
layer.backward([bottom], [top], True)
np.testing.assert_array_equal(top.data(), bottom.data())
np.testing.assert_array_equal(top.diff(), bottom.diff())
def testSquaredLossGrad(self):
np.random.seed(1701)
shapes = [(4, 3), (1, 10), (4, 3, 2)]
layer = core_layers.SquaredLossLayer(name='squared')
checker = gradcheck.GradChecker(1e-6)
for shape in shapes:
input_blob = base.Blob(shape, filler=fillers.GaussianRandFiller())
target_blob = base.Blob(shape, filler=fillers.GaussianRandFiller())
result = checker.check(layer, [input_blob, target_blob], [],
check_indices=[0])
print(result)
self.assertTrue(result[0])
# also, check if weight works.
self._testWeight(layer, [input_blob, target_blob])
def testSoftmaxGrad(self):
np.random.seed(1701)
input_blob = base.Blob((10,5), filler=fillers.GaussianRandFiller())
output_blob = base.Blob()
layer = core_layers.SoftmaxLayer(name='softmax')
checker = gradcheck.GradChecker(1e-5)
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
# Also, let's check the result
pred = input_blob.data()
prob = np.exp(pred) / np.exp(pred).sum(1)[:, np.newaxis]
np.testing.assert_array_almost_equal(
output_blob.data(), prob)
def testPaddingGrad(self):
np.random.seed(1701)
output_blob = base.Blob()
checker = gradcheck.GradChecker(1e-5)
shapes = [(1, 5, 5, 1), (1, 5, 5, 3), (1, 4, 3, 1), (1, 4, 3, 3)]
pads = [1, 2, 3]
for pad in pads:
for shape in shapes:
input_blob = base.Blob(shape,
filler=fillers.GaussianRandFiller())
layer = core_layers.PaddingLayer(name='padding', pad=pad)
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
def testIm2colGrad(self):
np.random.seed(1701)
output_blob = base.Blob()
checker = gradcheck.GradChecker(1e-4)
shapes = [(1, 5, 5, 1), (1, 5, 5, 3), (1, 4, 3, 1), (1, 4, 3, 3)]
params = [(2, 1), (2, 2), (3, 1), (3, 2)]
for psize, stride in params:
for shape in shapes:
input_blob = base.Blob(shape,
filler=fillers.GaussianRandFiller())
layer = core_layers.Im2colLayer(name='im2col',
psize=psize,
stride=stride)
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
def whitened_images(dtype=np.float64):
"""Returns the whitened images provided in the Sparsenet website:
http://redwood.berkeley.edu/bruno/sparsenet/
The returned data will be in the shape (10,512,512,1) to fit
the blob convension.
"""
npzdata = np.load(os.path.join(_DATA_PATH, 'whitened_images.npz'))
blob = base.Blob(npzdata['images'].shape, dtype)
blob.data().flat = npzdata['images'].flat
return blob
def testPoolingGrad(self):
np.random.seed(1701)
output_blob = base.Blob()
checker = gradcheck.GradChecker(1e-4)
shapes = [(1, 7, 7, 1), (2, 7, 7, 1), (1, 7, 7, 3), (1, 8, 8, 3),
(1, 13, 13, 1), (1, 13, 13, 2)]
params = [(3, 2, 'max'), (3, 2, 'ave'), (3, 3, 'max'), (3, 3, 'ave'),
(5, 3, 'max'), (5, 3, 'ave'), (5, 5, 'max'), (5, 5, 'ave')]
for shape in shapes:
for psize, stride, mode in params:
print(psize, stride, mode, shape)
input_blob = base.Blob(shape,
filler=fillers.GaussianRandFiller())
layer = core_layers.PoolingLayer(name='pool',
psize=psize,
stride=stride,
mode=mode)
result = checker.check(layer, [input_blob], [output_blob])
print(result)
self.assertTrue(result[0])
def __init__(self, **kwargs):
"""Initializes a Dropout layer.
kwargs:
name: the layer name.
ratio: the ratio to carry out dropout.
debug_freeze: a debug flag. If set True, the mask will only
be generated once when running. You should not use it other
than purposes like gradient check.
"""
base.Layer.__init__(self, **kwargs)
filler = fillers.DropoutFiller(ratio=self.spec['ratio'])
self._mask = base.Blob(filler=filler)
def testPoolingGrad(self):
np.random.seed(1701)
output_blob = base.Blob()
input_blob = base.Blob((1, 8, 8, 3),
filler=fillers.GaussianRandFiller())
psize = 3
stride = 2
mode = 'max'
layer = core_layers.PoolingLayer(name='pool',
psize=psize,
stride=stride,
mode=mode)
layer.forward([input_blob], [output_blob])
img = input_blob.data()[0]
output = output_blob.data()[0]
print img.shape, output.shape
for i in range(output.shape[0]):
for j in range(output.shape[1]):
for c in range(output.shape[2]):
self.assertAlmostEqual(
output[i, j, c],
img[i * stride:i * stride + psize,
j * stride:j * stride + psize, c].max())
mode = 'ave'
layer = core_layers.PoolingLayer(name='pool',
psize=psize,
stride=stride,
mode=mode)
layer.forward([input_blob], [output_blob])
img = input_blob.data()[0]
output = output_blob.data()[0]
print img.shape, output.shape
for i in range(output.shape[0]):
for j in range(output.shape[1]):
for c in range(output.shape[2]):
self.assertAlmostEqual(
output[i, j, c],
img[i * stride:i * stride + psize,
j * stride:j * stride + psize, c].mean())
