class ConvolutionalActivation(Initializable):
"""A convolution followed by an activation function.
Parameters
----------
activation : :class:`.BoundApplication`
The application method to apply after convolution (i.e.
the nonlinear activation function)
See Also
--------
:class:`Convolutional` : For the documentation of other parameters.
"""
@lazy(allocation=['filter_size', 'num_filters', 'num_channels'])
def __init__(self, activation, filter_size, num_filters, num_channels,
batch_size=None, image_size=None, step=(1, 1),
border_mode='valid', tied_biases=False, **kwargs):
self.convolution = Convolutional(name='conv'+ kwargs['name'])
self.bn = BatchNorm(name='bn'+ kwargs['name'])
self.activation = activation
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.batch_size = batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
super(ConvolutionalActivation, self).__init__(**kwargs)
self.children = [self.convolution, self.bn, self.activation]
def _push_allocation_config(self):
for attr in ['filter_size', 'num_filters', 'step', 'border_mode',
'batch_size', 'num_channels', 'image_size',
'tied_biases']:
setattr(self.convolution, attr, getattr(self, attr))
setattr(self.bn, 'input_dim', self.num_filters)
def get_dim(self, name):
# TODO The name of the activation output doesn't need to be `output`
return self.convolution.get_dim(name)
def apply(self, input_):
out = self.convolution.apply(input_)
out = self.bn.apply(out)
out = self.activation.apply(out)
return out
def inference(self, input_):
out = self.convolution.apply(input_)
out = self.bn.inference(out)
out = self.activation.apply(out)
return out
def __init__(self, activation, filter_size, num_filters, num_channels,
batch_size=None, image_size=None, step=(1, 1),
border_mode='valid', tied_biases=False, **kwargs):
self.convolution = Convolutional(name='conv'+ kwargs['name'])
self.bn = BatchNorm(name='bn'+ kwargs['name'])
self.activation = activation
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.batch_size = batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
super(ConvolutionalActivation, self).__init__(**kwargs)
self.children = [self.convolution, self.bn, self.activation]
def init_var(self):
self._has_init_var = True
dims = self.dims
wd = self.wd
with tf.variable_scope(self.scope):
for ii in xrange(self.nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
nin = dims[ii]
nout = dims[ii + 1]
if self.init_weights is not None and \
self.init_weights[ii] is not None:
init_val_w = self.init_weights[ii]['w']
init_val_b = self.init_weights[ii]['b']
else:
init_val_w = None
init_val_b = None
if self.frozen is not None and self.frozen[ii]:
trainable = False
else:
trainable = True
if self.use_bn[ii]:
self.bn[ii] = BatchNorm(nin, [0], decay=0.999)
if self.init_stddev is None:
self.log.info('Using Xavier initialization')
std = 1 / np.sqrt(nin)
self.log.info('Std: {:.4f}'.format(std))
self.log.info('Uniform')
dist = 'uniform'
else:
self.log.info('Using standard initialization')
std = self.init_stddev
self.log.info('Std: {:.4f}'.format(std))
self.log.info('Gaussian')
dist = 'normal'
self.w[ii] = self.declare_var([nin, nout],
init_val=init_val_w,
wd=wd,
name='w',
trainable=trainable,
stddev=std,
dist=dist)
self.log.info('Weights: {} Trainable: {}'.format(
[nin, nout], trainable))
if self.add_bias:
self.b[ii] = self.declare_var([nout],
init_val=init_val_b,
name='b',
trainable=trainable)
self.log.info('Bias: {} Trainable: {}'.format(
[nout], trainable))
pass
pass
pass
pass
pass
def build(self, inp):
self.lazy_init_var()
x = inp['input']
phase_train = inp['phase_train']
skip = inp['skip']
with tf.variable_scope(self.scope):
h = [None] * self.nlayers
out_shape = [None] * self.nlayers
batch = tf.shape(x)[0:1]
inp_size = tf.shape(x)[1:3]
cum_pool = 1
for ii in xrange(self.nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
cum_pool *= self.pool[ii]
out_ch = self.channels[ii + 1]
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
if skip is not None:
if skip[ii] is not None:
if ii == 0:
prev_inp = tf.concat(3, [prev_inp, skip[ii]])
else:
prev_inp = tf.concat(3, [prev_inp, skip[ii]])
out_shape[ii] = tf.concat(
0, [batch, inp_size * cum_pool,
tf.constant([out_ch])])
h[ii] = tf.nn.conv2d_transpose(
prev_inp,
self.w[ii],
out_shape[ii],
strides=[1, self.pool[ii], self.pool[ii], 1
]) + self.b[ii]
if self.use_bn[ii]:
self.batch_norm[self.num_copies][ii] = BatchNorm(
out_ch)
h[ii] = self.batch_norm[self.num_copies][ii]({
'input':
h[ii],
'phase_train':
phase_train
})
if self.act[ii] is not None:
h[ii] = self.act[ii](h[ii])
self.num_copies += 1
self.hidden_layers = h
return h[-1]
def build(self, inp):
"""Run CNN on an input.
Args:
input: input image, [B, H, W, D]
phase_train: phase train, bool
"""
self.lazy_init_var()
x = inp['input']
phase_train = inp['phase_train']
h = [None] * self.nlayers
self.batch_norm.append([None] * self.nlayers)
with tf.variable_scope(self.scope):
for ii in xrange(self.nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
out_ch = self.channels[ii + 1]
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
h[ii] = Conv2D(self.w[ii])(prev_inp) + self.b[ii]
if self.use_bn[ii]:
self.batch_norm[self.num_copies][ii] = BatchNorm(
out_ch)
h[ii] = self.batch_norm[self.num_copies][ii]({
'input':
h[ii],
'phase_train':
phase_train
})
if self.act[ii] is not None:
h[ii] = self.act[ii](h[ii])
if self.pool[ii] > 1:
h[ii] = MaxPool(self.pool[ii])(h[ii])
pass
# h[ii] = tf.Print(h[ii], [ii, tf.reduce_mean(h[ii])])
pass
pass
self.num_copies += 1
self.hidden_layers = h
return h[-1]
def apply_shortcut(self,
prev_inp,
ch_in,
ch_out,
phase_train=None,
w=None,
stride=None):
if self.shortcut == 'projection':
if self.dilation:
prev_inp = DilatedConv2D(w, rate=stride)(prev_inp)
else:
prev_inp = Conv2D(w, stride=stride)(prev_inp)
bn = BatchNorm(ch_out)
prev_inp = bn({'input': prev_inp, 'phase_train': phase_train})
elif self.shortcut == 'identity':
pad_ch = ch_out - ch_in
if pad_ch < 0:
raise Exception('Must use projection when ch_in > ch_out.')
prev_inp = tf.pad(prev_inp, [[0, 0], [0, 0], [0, 0], [0, pad_ch]])
if stride > 1:
prev_inp = AvgPool(stride)(prev_inp)
bn = None
self.log.info('After proj shape: {}'.format(prev_inp.get_shape()))
return prev_inp, bn
def init_var(self):
with tf.variable_scope(self.scope):
for ii in xrange(self.num_stage):
ch_in = self.channels[ii]
ch_out = self.channels[ii + 1]
with tf.variable_scope('stage_{}'.format(ii)):
self.w[ii] = [None] * self.layers[ii]
self.bn[ii] = [None] * self.layers[ii]
# self.b[ii] = [None] * self.layers[ii]
for jj in xrange(self.layers[ii]):
if jj > 0:
ch_in = ch_out
pass
self.w[ii][jj] = [None] * self.unit_depth
self.bn[ii][jj] = [None] * self.unit_depth
if jj == 0 and (ch_in != ch_out or self.compatible):
if self.shortcut == 'projection':
with tf.variable_scope('shortcut'):
self.shortcut_w[ii] = self.declare_var(
[1, 1, ch_in, ch_out], wd=self.wd,
name='w',
stddev=self.compute_std(
[1, 1, ch_in, ch_out]),
trainable=self.trainable
)
self.shortcut_bn[ii] = BatchNorm(
ch_out, trainable=self.trainable)
pass
pass
with tf.variable_scope('layer_{}'.format(jj)):
for kk in xrange(self.unit_depth):
with tf.variable_scope('unit_{}'.format(kk)):
f_, ch_in_, ch_out_ = self.compute_in_out(
kk, ch_in, ch_out)
self.w[ii][jj][kk] = self.declare_var(
[f_, f_, ch_in_, ch_out_], wd=self.wd,
name='w',
stddev=self.compute_std(
[f_, f_, ch_in_, ch_out_]),
trainable=self.trainable
)
if self.compatible:
bn_ch = ch_out_
else:
bn_ch = ch_in_
self.bn[ii][jj][kk] = BatchNorm(
bn_ch, trainable=self.trainable)
self.log.info('Init SD: {}'.format(
self.compute_std(
[f_, f_, ch_in_, ch_out_])))
self.log.info('Filter: {}'.format(
[f_, f_, ch_in_, ch_out_]))
self.log.info('Weights: {}'.format(
self.w[ii][jj][kk].name))
pass
pass
pass
pass
pass
pass
pass
pass
pass
return results
pass
if __name__ == '__main__':
# 34-layer ResNet
x = tf.placeholder('float', [3, 224, 224, 3], name='x')
phase_train = tf.placeholder('bool', name='phase_train')
w1 = tf.Variable(tf.truncated_normal_initializer(
stddev=0.01)([7, 7, 3, 64]), name='w1')
print w1
b1 = tf.Variable(tf.truncated_normal_initializer(
stddev=0.01)([64]), name='b1')
h1 = Conv2D(w1, stride=2)(x) + b1
bn1 = BatchNorm(64)
h1 = bn1({'input': h1, 'phase_train': phase_train})
h1 = MaxPool(2)(h1)
hn = ResNet(layers=[3, 4, 6, 3],
bottleneck=True,
shortcut='projection',
channels=[64, 64, 128, 256, 512],
strides=[1, 2, 2, 2])(
{'input': h1, 'phase_train': phase_train})
print hn.get_shape()
hn = AvgPool(7)(hn)
print hn.get_shape()
hn = ResNet(layers=[3, 4, 6, 3],
bottleneck=False,
def build(self, inp):
self.lazy_init_var()
x = inp['input']
phase_train = inp['phase_train']
prev_inp = x
with tf.variable_scope(self.scope):
for ii in xrange(self.num_stage):
print 'Stage count', ii, 'of', self.num_stage
ch_in = self.channels[ii]
ch_out = self.channels[ii + 1]
s = self.strides[ii]
with tf.variable_scope('stage_{}'.format(ii)):
self.bn[ii] = [None] * self.layers[ii]
print 'H1'
for jj in xrange(self.layers[ii]):
print 'Layer count', jj, 'of', self.layers[ii]
h = prev_inp
if jj > 0:
ch_in = ch_out
else:
# First unit of the layer.
print 'In', ch_in, 'Out', ch_out
if self.compatible:
# In compatible mode, always project.
with tf.variable_scope('shortcut'):
prev_inp, proj_bn = self.apply_shortcut(
prev_inp,
ch_in,
ch_out,
phase_train=phase_train,
w=self.shortcut_w[ii],
stride=s)
self.shortcut_bn[ii] = proj_bn
else:
if ch_in != ch_out:
with tf.variable_scope('shortcut'):
prev_inp, proj_bn = self.apply_shortcut(
prev_inp,
ch_in,
ch_out,
phase_train=phase_train,
w=self.shortcut_w[ii],
stride=s)
self.shortcut_bn[ii] = proj_bn
elif s != 1:
if not self.dilation:
prev_inp = AvgPool(s)(prev_inp)
self.log.info(
'After pool shape: {}'.format(
prev_inp.get_shape()))
with tf.variable_scope('layer_{}'.format(jj)):
self.bn[ii][jj] = [None] * self.unit_depth
if self.compatible:
# A compatible graph for building weights
# older version of ResNet
if self.bottleneck:
with tf.variable_scope('unit_0'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
0, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][0],
stride=s)(h)
self.bn[ii][jj][0] = BatchNorm(
ch_out_, trainable=self.trainable)
h = self.bn[ii][jj][0]({
'input':
h,
'phase_train':
phase_train
})
h = tf.nn.relu(h)
self.register_var(
'stage_{}/layer_{}/unit_{}/relu'.
format(ii, jj, 0), h)
with tf.variable_scope('unit_1'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
1, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][1])(h)
self.bn[ii][jj][1] = BatchNorm(
ch_out_, trainable=self.trainable)
h = self.bn[ii][jj][1]({
'input':
h,
'phase_train':
phase_train
})
h = tf.nn.relu(h)
print 'stage_{}/layer_{}/unit_{}/relu'.format(
ii, jj, 1)
self.register_var(
'stage_{}/layer_{}/unit_{}/relu'.
format(ii, jj, 1), h)
with tf.variable_scope('unit_2'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
2, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][2])(h)
self.bn[ii][jj][2] = BatchNorm(
ch_out_, trainable=self.trainable)
h = self.bn[ii][jj][2]({
'input':
h,
'phase_train':
phase_train
})
print 'stage_{}/layer_{}/unit_{}/relu'.format(
ii, jj, 2)
self.register_var(
'stage_{}/layer_{}/unit_{}/relu'.
format(ii, jj, 2), h)
else:
with tf.variable_scope('unit_0'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
0, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][0],
stride=s)(h)
self.bn[ii][jj][0] = BatchNorm(
ch_out_, trainable=self.trainable)
h = self.bn[ii][jj][0]({
'input':
h,
'phase_train':
phase_train
})
h = tf.nn.relu(h)
with tf.variable_scope('unit_1'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
1, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][1])(h)
self.bn[ii][jj][1] = BatchNorm(
ch_out_, trainable=self.trainable)
h = self.bn[ii][jj][1]({
'input':
h,
'phase_train':
phase_train
})
s = 1
else:
# New version of ResNet
# Full pre-activation
for kk in xrange(self.unit_depth):
with tf.variable_scope(
'unit_{}'.format(kk)):
f_, ch_in_, ch_out_ = self.compute_in_out(
kk, ch_in, ch_out)
self.bn[ii][jj][kk] = BatchNorm(
ch_in_, trainable=self.trainable)
h = self.bn[ii][jj][kk]({
'input':
h,
'phase_train':
phase_train
})
h = tf.nn.relu(h)
if self.dilation:
h = DilatedConv2D(
self.w[ii][jj][kk], rate=s)(h)
else:
h = Conv2D(self.w[ii][jj][kk],
stride=s)(h)
self.log.info(
'Unit {} shape: {}'.format(
kk, h.get_shape()))
pass
if not self.dilation and kk == 0:
s = 1
pass
if self.compatible:
# Old version
# Relu after add
prev_inp = tf.nn.relu(prev_inp + h)
else:
# New version
# Pure linear
prev_inp = prev_inp + h
self.log.info('After add shape: {}'.format(
prev_inp.get_shape()))
pass
pass
pass
pass
return prev_inp
# plt.imshow(fg_images[index])
# plt.show()
########### holders
ph = placeHolders(input_images=rgb_images, input_labels=fg_images)
########### layer
# first step
# get_shape("input data: {}", input_data)
gen_convolution = method.layers_deeplab(method.TYPE_NORMAL,
ph.input_data,
64,
"layer1",
method.FUNC_RELU,
BatchNorm(is_train=ph.is_train,
use_batch_norm=True),
pooling=None)
gen_convolution = method.layers_deeplab(method.TYPE_NORMAL,
gen_convolution,
64,
"layer2_pooling",
method.FUNC_RELU,
BatchNorm(is_train=ph.is_train,
use_batch_norm=True),
pooling={
'size': 2,
'stride': 2
})
# step 2
#
# plt.imshow(rgb_images[index])
# plt.show()
# plt.imshow(fg_images[index])
# plt.show()
########### holders
ph = placeHolders(input_images=rgb_images, input_labels=fg_images)
########### layer
# first step
# get_shape("input data: {}", input_data)
gen_convolution = method.layers(method.TYPE_NORMAL, ph.input_data, 64, "layer1", method.FUNC_RELU,
BatchNorm(is_train=ph.is_train, use_batch_norm=True), pooling=None)
gen_convolution = method.layers(method.TYPE_NORMAL, gen_convolution, 64, "layer2_pooling", method.FUNC_RELU,
BatchNorm(is_train=ph.is_train, use_batch_norm=True), pooling={'size': 2, 'stride': 2})
# step 2
gen_convolution = method.layers(method.TYPE_NORMAL, gen_convolution, 128, "layer3", method.FUNC_RELU,
BatchNorm(is_train=ph.is_train, use_batch_norm=True), pooling=None)
gen_convolution = method.layers(method.TYPE_NORMAL, gen_convolution, 128, "layer3_1", method.FUNC_RELU,
BatchNorm(is_train=ph.is_train, use_batch_norm=True), pooling=None)
gen_convolution = method.layers(method.TYPE_ATROUS, gen_convolution, 128, "layer4_pooling", method.FUNC_RELU,
BatchNorm(is_train=ph.is_train, use_batch_norm=True), pooling={'size': 2, 'stride': 2})
# step 3
gen_convolution = method.layers(method.TYPE_NORMAL, gen_convolution, 256, "layer5", method.FUNC_RELU,
BatchNorm(is_train=ph.is_train, use_batch_norm=True), pooling=None)