def neural_net(self):
with tf.name_scope('input'):
self.global_step = tf.Variable(0, trainable=False)
self.drop_rate = tf.placeholder(tf.float32)
self.training = tf.placeholder(tf.bool)
self.X = tf.placeholder(tf.float32, [None, 32, 32, 3], name='X')
self.Y = tf.placeholder(tf.int32, [None, 10], name='Y')
with tf.name_scope('layer1') as scope:
X = layer.conv2D('conv1-1', self.X, 30, [7, 7], [1, 1], 'same') # 32 x 32
X = layer.BatchNorm('BN1-1', X, self.training)
X = layer.p_relu('active1-1', X)
print(X.shape)
X = layer.maxpool('mp1', X, [2, 2], [2, 2]) # 16 x 16
print(X.shape)
with tf.name_scope('layer2') as scope:
X = layer.conv2D('conv2-1', X, 60, [3, 3], [1, 1], 'same') # 16 x 16
X = layer.BatchNorm('BN2-1', X, self.training)
X = layer.p_relu('active2-1', X)
print(X.shape)
X = layer.maxpool('mp2', X, [2, 2], [2, 2]) # 8 x 8
print(X.shape)
with tf.name_scope('layer3') as scope:
X = layer.conv2D('conv3-1', X, 120, [3, 3], [1, 1], 'same') # 8 x 8
X = layer.BatchNorm('BN3-1', X, self.training)
X = layer.p_relu('active3-1', X)
print(X.shape)
# X = layer.maxpool('mp3', X, [2, 2], [2, 2]) # 4 x 4
with tf.name_scope('bottleneck1') as scope:
RX = layer.conv2D('rconv1-1', X, 120, [1, 1], [1, 1], 'same')
RX = layer.BatchNorm('rBN1-1', RX, self.training)
X = layer.conv2D('bconv1-1', X, 60, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN1-1', X, self.training)
X = layer.p_relu('bactive1-1', X)
X = layer.conv2D('bconv1-2', X, 60, [3, 3], [1, 1], 'same')
X = layer.BatchNorm('bBN1-2', X, self.training)
X = layer.p_relu('bactive1-2', X)
# X = layer.dropout('dp1', X, self.drop_rate, self.training)
X = layer.conv2D('bconv1-3', X, 120, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN1-3', X, self.training)
X = layer.add(X, RX, name='add1')
X = layer.p_relu('bactive1-3', X)
print(X.shape)
with tf.name_scope('bottleneck2') as scope:
RX = layer.conv2D('rconv2-1', X, 240, [1, 1], [1, 1], 'same')
RX = layer.BatchNorm('rBN2-1', RX, self.training)
X = layer.conv2D('bconv2-1', X, 120, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN2-1', X, self.training)
X = layer.p_relu('bactive2-1', X)
X = layer.conv2D('bconv2-2', X, 120, [3, 3], [1, 1], 'same')
X = layer.BatchNorm('bBN2-2', X, self.training)
X = layer.p_relu('bactive2-2', X)
# X = layer.dropout('dp2', X, self.drop_rate, self.training)
X = layer.conv2D('bconv2-3', X, 240, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN2-3', X, self.training)
X = layer.add(X, RX, name='add2')
X = layer.p_relu('bactive2-3', X)
print(X.shape)
with tf.name_scope('bottleneck3') as scope:
RX = layer.conv2D('rconv3-1', X, 360, [1, 1], [1, 1], 'same')
RX = layer.BatchNorm('rBN3-1', RX, self.training)
X = layer.conv2D('bconv3-1', X, 240, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN3-1', X, self.training)
X = layer.p_relu('bactive3-1', X)
X = layer.conv2D('bconv3-2', X, 240, [3, 3], [1, 1], 'same')
X = layer.BatchNorm('bBN3-2', X, self.training)
X = layer.p_relu('bactive3-2', X)
# X = layer.dropout('dp3', X, self.drop_rate, self.training)
X = layer.conv2D('bconv3-3', X, 360, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN3-3', X, self.training)
X = layer.add(X, RX, name='add3')
X = layer.p_relu('bactive3-3', X)
print(X.shape)
with tf.name_scope('bottleneck4') as scope:
RX = layer.conv2D('rconv4-1', X, 480, [1, 1], [1, 1], 'same')
RX = layer.BatchNorm('rBN4-1', RX, self.training)
X = layer.conv2D('bconv4-1', X, 360, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN4-1', X, self.training)
X = layer.p_relu('bactive4-1', X)
X = layer.conv2D('bconv4-2', X, 360, [3, 3], [1, 1], 'same')
X = layer.BatchNorm('bBN4-2', X, self.training)
X = layer.p_relu('bactive4-2', X)
# X = layer.dropout('dp4', X, self.drop_rate, self.training)
X = layer.conv2D('bconv4-3', X, 480, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN4-3', X, self.training)
X = layer.add(X, RX, name='add4')
X = layer.p_relu('bactive4-3', X)
print(X.shape)
'''
with tf.name_scope('bottleneck5') as scope:
RX = layer.conv2D('rconv5-1', X, 720, [1, 1], [1, 1], 'same')
RX = layer.BatchNorm('rBN5-1', RX, self.training)
X = layer.conv2D('bconv5-1', X, 480, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN5-1', X, self.training)
X = layer.p_relu('bactive5-1', X)
X = layer.conv2D('bconv5-2', X, 480, [3, 3], [1, 1], 'same')
X = layer.BatchNorm('bBN5-2', X, self.training)
X = layer.p_relu('bactive5-2', X)
X = layer.dropout('dp5', X, self.drop_rate, self.training)
X = layer.conv2D('bconv5-3', X, 720, [1, 1], [1, 1], 'same')
X = layer.BatchNorm('bBN5-3', X, self.training)
X = layer.add(X, RX, name='add5')
X = layer.p_relu('bactive5-3', X)
print(X.shape)
'''
with tf.name_scope('GAP') as scope:
X = layer.conv2D('GAP_1', X, 10, [1, 1], [1, 1], 'same')
print(X.shape)
X = layer.averagepool('avp', X, [8, 8], [1, 1])
print(X.shape)
self.logits = tf.squeeze(X, [1, 2])
print(self.logits.shape)
with tf.name_scope('optimizer'):
self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.logits, labels=self.Y))
self.l2_loss = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
self.loss = tf.add(self.loss, self.l2_loss)
self.init_learning = 0.01
self.decay_step = 2500
self.decay_rate = 0.9
self.exponential_decay_learning_rate = tf.train.exponential_decay(
learning_rate=self.init_learning,
global_step=self.global_step,
decay_steps=self.decay_step,
decay_rate=self.decay_rate,
staircase=True,
name='learning_rate'
)
self.optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(self.update_ops):
self.trainer = self.optimizer.minimize(self.loss, global_step=self.global_step, name='train')
self.accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1)), tf.float32))
tf.summary.scalar('loss', self.loss)
tf.summary.scalar('lr', self.exponential_decay_learning_rate)
tf.summary.scalar('accuracy', self.accuracy)
self.merged = tf.summary.merge_all()
self.writer = tf.summary.FileWriter('./logs', self.sess.graph)
def neural_net(self):
with tf.name_scope('input'):
self.global_step = tf.Variable(0, trainable=False)
self.drop_rate = tf.placeholder(tf.float32)
self.training = tf.placeholder(tf.bool)
self.X = tf.placeholder(tf.float32, [None, 32, 32, 3], name='X')
self.Y = tf.placeholder(tf.int32, [None, 10], name='Y')
with tf.name_scope('layer1'):
self.layer = layer.conv2D('conv1-1', self.X, 30, [1, 5], [1, 1])
self.layer = layer.BatchNorm('bn1', self.layer, self.training)
self.layer = layer.p_relu('active1-1', self.layer)
# print(self.layer.shape)
self.layer = layer.conv2D('conv1-2', self.layer, 30, [5, 1],
[1, 1])
self.layer = layer.BatchNorm('bn2', self.layer, self.training)
self.layer = layer.p_relu('active1-2', self.layer)
# print(self.layer.shape)
self.layer = layer.maxpool('mp1', self.layer, [2, 2], [2, 2])
# print(self.layer.shape)
with tf.name_scope('layer2'):
self.layer = layer.conv2D('conv2-1', self.layer, 90, [1, 3],
[1, 1])
self.layer = layer.BatchNorm('bn3', self.layer, self.training)
self.layer = layer.p_relu('active2-1', self.layer)
# print(self.layer.shape)
self.layer = layer.conv2D('conv2-2', self.layer, 90, [3, 1],
[1, 1])
self.layer = layer.BatchNorm('bn4', self.layer, self.training)
self.layer = layer.p_relu('active2-2', self.layer)
# print(self.layer.shape)
self.layer = layer.maxpool('mp2', self.layer, [2, 2], [2, 2])
# print(self.layer.shape)
with tf.name_scope('layer3'):
self.layer = layer.conv2D('conv3-1', self.layer, 270, [1, 2],
[1, 1])
self.layer = layer.BatchNorm('bn5', self.layer, self.training)
self.layer = layer.p_relu('active3-1', self.layer)
# print(self.layer.shape)
self.layer = layer.conv2D('conv3-2', self.layer, 270, [2, 1],
[1, 1])
self.layer = layer.BatchNorm('bn6', self.layer, self.training)
self.layer = layer.p_relu('active3-2', self.layer)
# print(self.layer.shape)
with tf.name_scope('middle_flow'):
self.m_layer = self.layer
for i in range(8):
self.residual = self.m_layer
self.m_layer = layer.s_conv2D('s_conv' + str(i) + '-1',
self.m_layer, 540, [1, 1],
[1, 1], 'same')
self.m_layer = layer.BatchNorm('bn' + str(i) + '1',
self.m_layer, self.training)
self.m_layer = layer.p_relu('m_active' + str(i) + '-1',
self.m_layer)
self.m_layer = layer.s_conv2D('s_conv' + str(i) + '-2',
self.m_layer, 540, [3, 3],
[1, 1], 'same')
self.m_layer = layer.BatchNorm('bn' + str(i) + '2',
self.m_layer, self.training)
self.m_layer = layer.p_relu('m_active' + str(i) + '-2',
self.m_layer)
self.m_layer = layer.dropout('m_dp', self.m_layer,
self.drop_rate, self.training)
self.m_layer = layer.s_conv2D('s_conv' + str(i) + '-3',
self.m_layer, 540, [3, 3],
[1, 1], 'same')
self.m_layer = layer.BatchNorm('bn' + str(i) + '3',
self.m_layer, self.training)
self.m_layer = layer.p_relu('m_active' + str(i) + '-3',
self.m_layer)
# print(self.m_layer.shape)
self.m_layer = layer.s_conv2D('s_conv' + str(i) + '-4',
self.m_layer, 270, [1, 1],
[1, 1], 'same')
self.m_layer = layer.add(self.m_layer,
self.residual,
name='add' + str(i))
self.m_layer = layer.p_relu('m_active' + str(i) + '-4',
self.m_layer)
# print(self.m_layer.shape)
with tf.name_scope('Global_Average_Pooling'):
self.layer = layer.s_conv2D('reduce_channel', self.m_layer, 10,
[1, 1], [1, 1])
# print(self.layer.shape)
self.layer = layer.averagepool('avp', self.layer, [5, 5], [1, 1])
# print(self.layer.shape)
self.logits = tf.squeeze(self.layer, [1, 2], name='logits')
# print(self.logits.shape)
with tf.name_scope('optimizer'):
self.loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,
labels=self.Y))
# self.l2_loss = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
# self.loss = tf.add(self.loss, self.l2_loss)
self.init_learning = 0.01
self.decay_step = 2500
self.decay_rate = 0.9
self.exponential_decay_learning_rate = tf.train.exponential_decay(
learning_rate=self.init_learning,
global_step=self.global_step,
decay_steps=self.decay_step,
decay_rate=self.decay_rate,
staircase=True,
name='learning_rate')
self.optimizer = tf.train.AdamOptimizer(
learning_rate=self.exponential_decay_learning_rate,
epsilon=0.0001)
self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(self.update_ops):
self.trainer = self.optimizer.minimize(
self.loss, global_step=self.global_step, name='train')
self.accuracy = tf.reduce_mean(
tf.cast(
tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1)),
tf.float32))
tf.summary.scalar('loss', self.loss)
tf.summary.scalar('lr', self.exponential_decay_learning_rate)
tf.summary.scalar('accuracy', self.accuracy)
self.merged = tf.summary.merge_all()
self.writer = tf.summary.FileWriter('./logs', self.sess.graph)
def create_separable_conv(
x,
x_float,
out_channels,
ksize,
stride=1,
qweight=False,
qactivation=False,
scale=None,
has_bn=True,
has_relu=True,
phase_train=False,
block_id=0,
):
in_channels = x.shape.as_list()[3]
depthwise_filters = in_channels
pointwise_filters = _make_divisible(out_channels, 8)
prefix = 'block_{}_'.format(block_id)
f = x
f_float = x_float
if block_id:
# Expand
f, f_float = conv_bn_relu(x,
x_float,
in_channels * 6,
1,
stride=1,
qweight=qweight,
qactivation=qactivation,
padding='SAME',
scale=scale,
has_bn=has_bn,
has_relu=has_relu,
phase_train=phase_train,
scope=prefix + 'expand')
depthwise_filters = in_channels * 6
f, f_float = conv_bn_relu(f,
f_float,
depthwise_filters,
ksize,
stride=stride,
qweight=qweight,
qactivation=qactivation,
padding='SAME',
groups=depthwise_filters,
scale=scale,
has_bn=has_bn,
has_relu=has_relu,
phase_train=phase_train,
scope=prefix + 'depthwise')
f, f_float = conv_bn_relu(f,
f_float,
pointwise_filters,
1,
stride=1,
qweight=qweight,
qactivation=qactivation,
padding='SAME',
scale=scale,
has_bn=has_bn,
has_relu=False,
phase_train=phase_train,
scope=prefix + 'project')
if in_channels == pointwise_filters and stride == 1:
f, f_float = add(f,
f_float,
x,
x_float,
phase_train,
qactivation=qactivation,
scope=prefix + 'add',
scale=scale)
return f, f_float