def __init__(self, key, prev_keys, hidden_units, choose_from_k,
cnn_keep_prob, lstm_keep_prob, att_keep_prob, att_mask):
super(Layer, self).__init__(key)
def conv_shortcut(kernel_size):
return ConvBN(kernel_size, hidden_units, hidden_units,
cnn_keep_prob, False, True)
self.n_candidates = len(prev_keys)
if self.n_candidates:
self.prec = mutables.InputChoice(
choose_from=prev_keys[-choose_from_k:], n_chosen=1)
else:
# first layer, skip input choice
self.prec = None
self.op = mutables.LayerChoice([
conv_shortcut(1),
conv_shortcut(3),
conv_shortcut(5),
conv_shortcut(7),
AvgPool(3, False, True),
MaxPool(3, False, True),
RNN(hidden_units, lstm_keep_prob),
Attention(hidden_units, 4, att_keep_prob, att_mask)
])
if self.n_candidates:
self.skipconnect = mutables.InputChoice(choose_from=prev_keys)
else:
self.skipconnect = None
self.bn = BatchNorm(hidden_units, False, True)
def apply_shortcut(self, prev_inp, ch_in, ch_out, phase_train=None, w=None, bn=None, stride=None):
if self.shortcut == 'projection':
if self.dilation:
prev_inp = DilatedConv2D(w, rate=stride)(prev_inp)
else:
prev_inp = Conv2D(w, stride=stride)(prev_inp)
prev_inp = bn({'input': prev_inp, 'phase_train': phase_train})
elif self.shortcut == 'identity':
pad_ch = ch_out - ch_in
if pad_ch < 0:
raise Exception('Must use projection when ch_in > ch_out.')
prev_inp = tf.pad(prev_inp, [[0, 0], [0, 0], [0, 0], [0, pad_ch]])
if stride > 1:
prev_inp = AvgPool(stride)(prev_inp)
raise Exception('DEBUG Unknown')
self.log.info('After proj shape: {}'.format(
prev_inp.get_shape()))
return prev_inp
def build(self, inp):
self.lazy_init_var()
x = inp['input']
phase_train = inp['phase_train']
prev_inp = x
with tf.variable_scope(self.scope):
for ii in xrange(self.num_stage):
self.log.info(
'Stage count {:d} of {:d}'.format(ii, self.num_stage))
ch_in = self.channels[ii]
ch_out = self.channels[ii + 1]
s = self.strides[ii]
with tf.variable_scope('stage_{}'.format(ii)):
for jj in xrange(self.layers[ii]):
self.log.info('Layer count {:d} of {:d}'.format(
jj, self.layers[ii]))
h = prev_inp
if jj > 0:
ch_in = ch_out
else:
# First unit of the layer.
self.log.info(
'In {:d} Out {:d}'.format(ch_in, ch_out))
if self.compatible:
# In compatible mode, always project.
with tf.variable_scope('shortcut'):
prev_inp = self.apply_shortcut(
prev_inp, ch_in, ch_out,
phase_train=phase_train,
w=self.shortcut_w[ii],
bn=self.shortcut_bn[ii],
stride=s)
else:
if ch_in != ch_out:
with tf.variable_scope('shortcut'):
prev_inp = self.apply_shortcut(
prev_inp, ch_in, ch_out,
phase_train=phase_train,
w=self.shortcut_w[ii],
bn=self.shortcut_bn[ii],
stride=s)
elif s != 1:
if not self.dilation:
prev_inp = AvgPool(s)(prev_inp)
self.log.info('After pool shape: {}'.format(
prev_inp.get_shape()))
with tf.variable_scope('layer_{}'.format(jj)):
if self.compatible:
# A compatible graph for building weights
# older version of ResNet
if self.bottleneck:
with tf.variable_scope('unit_0'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
0, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][0],
stride=s)(h)
h = self.bn[ii][jj][0](
{'input': h,
'phase_train': phase_train})
h = tf.nn.relu(h)
with tf.variable_scope('unit_1'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
1, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][1])(h)
h = self.bn[ii][jj][1](
{'input': h,
'phase_train': phase_train})
h = tf.nn.relu(h)
with tf.variable_scope('unit_2'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
2, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][2])(h)
h = self.bn[ii][jj][2](
{'input': h,
'phase_train': phase_train})
else:
with tf.variable_scope('unit_0'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
0, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][0],
stride=s)(h)
h = self.bn[ii][jj][0](
{'input': h,
'phase_train': phase_train})
h = tf.nn.relu(h)
with tf.variable_scope('unit_1'):
f_, ch_in_, ch_out_ = \
self.compute_in_out(
1, ch_in, ch_out)
h = Conv2D(self.w[ii][jj][1])(h)
h = self.bn[ii][jj][1](
{'input': h,
'phase_train': phase_train})
s = 1
else:
# New version of ResNet
# Full pre-activation
for kk in xrange(self.unit_depth):
with tf.variable_scope('unit_{}'.format(kk)):
f_, ch_in_, ch_out_ = self.compute_in_out(
kk, ch_in, ch_out)
h = self.bn[ii][jj][kk](
{'input': h,
'phase_train': phase_train})
h = tf.nn.relu(h)
if self.dilation:
h = DilatedConv2D(
self.w[ii][jj][kk], rate=s)(h)
else:
h = Conv2D(self.w[ii][jj][kk],
stride=s)(h)
self.log.info('Unit {} shape: {}'.format(
kk, h.get_shape()))
pass
if not self.dilation and kk == 0:
s = 1
pass
if self.compatible:
# Old version
# Relu after add
prev_inp = tf.nn.relu(prev_inp + h)
# self.register_var(
# 'stage_{}/layer_{}/relu'.format(ii, jj), prev_inp)
else:
# New version
# Pure linear
prev_inp = prev_inp + h
self.log.info('After add shape: {}'.format(
prev_inp.get_shape()))
pass
pass
pass
pass
return prev_inp
print w1
b1 = tf.Variable(tf.truncated_normal_initializer(
stddev=0.01)([64]), name='b1')
h1 = Conv2D(w1, stride=2)(x) + b1
bn1 = BatchNorm(64)
h1 = bn1({'input': h1, 'phase_train': phase_train})
h1 = MaxPool(2)(h1)
hn = ResNet(layers=[3, 4, 6, 3],
bottleneck=True,
shortcut='projection',
channels=[64, 64, 128, 256, 512],
strides=[1, 2, 2, 2])(
{'input': h1, 'phase_train': phase_train})
print hn.get_shape()
hn = AvgPool(7)(hn)
print hn.get_shape()
hn = ResNet(layers=[3, 4, 6, 3],
bottleneck=False,
shortcut='projection',
channels=[64, 64, 128, 256, 512],
strides=[1, 2, 2, 2])(
{'input': h1, 'phase_train': phase_train})
print hn.get_shape()
hn = AvgPool(7)(hn)
print hn.get_shape()
hn = ResNet(layers=[3, 4, 6, 3],
bottleneck=False,
dilation=True,