def _build_model(self, **kwargs):
d = dict()
num_classes = self.num_classes
pretrain = kwargs.pop('pretrain', True)
frontend = kwargs.pop('frontend', 'resnet_v2_50')
num_anchors = kwargs.pop('num_anchors', 9)
if pretrain:
frontend_dir = os.path.join('pretrained_models',
'{}.ckpt'.format(frontend))
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits, end_points = resnet_v2.resnet_v2_50(
self.X, is_training=self.is_train)
d['init_fn'] = slim.assign_from_checkpoint_fn(
model_path=frontend_dir,
var_list=slim.get_model_variables(frontend))
convs = [
end_points[frontend + '/block{}'.format(x)] for x in [4, 2, 1]
]
else:
#TODO build convNet
raise NotImplementedError("Build own convNet!")
with tf.variable_scope('layer5'):
d['s_5'] = conv_layer(convs[0], 256, (1, 1), (1, 1))
d['cls_head5'] = build_head_cls(d['s_5'], num_anchors,
num_classes + 1)
d['loc_head5'] = build_head_loc(d['s_5'], num_anchors)
d['flat_cls_head5'] = tf.reshape(
d['cls_head5'],
(tf.shape(d['cls_head5'])[0], -1, num_classes + 1))
d['flat_loc_head5'] = tf.reshape(
d['loc_head5'], (tf.shape(d['loc_head5'])[0], -1, 4))
with tf.variable_scope('layer6'):
d['s_6'] = conv_layer(d['s_5'], 256, (3, 3), (2, 2))
d['cls_head6'] = build_head_cls(d['s_6'], num_anchors,
num_classes + 1)
d['loc_head6'] = build_head_loc(d['s_6'], num_anchors)
d['flat_cls_head6'] = tf.reshape(
d['cls_head6'],
(tf.shape(d['cls_head6'])[0], -1, num_classes + 1))
d['flat_loc_head6'] = tf.reshape(
d['loc_head6'], (tf.shape(d['loc_head6'])[0], -1, 4))
with tf.variable_scope('layer7'):
d['s_7'] = conv_layer(tf.nn.relu(d['s_6']), 256, (3, 3), (2, 2))
d['cls_head7'] = build_head_cls(d['s_7'], num_anchors,
num_classes + 1)
d['loc_head7'] = build_head_loc(d['s_7'], num_anchors)
d['flat_cls_head7'] = tf.reshape(
d['cls_head7'],
(tf.shape(d['cls_head7'])[0], -1, num_classes + 1))
d['flat_loc_head7'] = tf.reshape(
d['loc_head7'], (tf.shape(d['loc_head7'])[0], -1, 4))
with tf.variable_scope('layer4'):
d['up4'] = resize_to_target(d['s_5'], convs[1])
d['s_4'] = conv_layer(convs[1], 256, (1, 1), (1, 1)) + d['up4']
d['cls_head4'] = build_head_cls(d['s_4'], num_anchors,
num_classes + 1)
d['loc_head4'] = build_head_loc(d['s_4'], num_anchors)
d['flat_cls_head4'] = tf.reshape(
d['cls_head4'],
(tf.shape(d['cls_head4'])[0], -1, num_classes + 1))
d['flat_loc_head4'] = tf.reshape(
d['loc_head4'], (tf.shape(d['loc_head4'])[0], -1, 4))
with tf.variable_scope('layer3'):
d['up3'] = resize_to_target(d['s_4'], convs[2])
d['s_3'] = conv_layer(convs[2], 256, (1, 1), (1, 1)) + d['up3']
d['cls_head3'] = build_head_cls(d['s_3'], num_anchors,
num_classes + 1)
d['loc_head3'] = build_head_loc(d['s_3'], num_anchors)
d['flat_cls_head3'] = tf.reshape(
d['cls_head3'],
(tf.shape(d['cls_head3'])[0], -1, num_classes + 1))
d['flat_loc_head3'] = tf.reshape(
d['loc_head3'], (tf.shape(d['loc_head3'])[0], -1, 4))
with tf.variable_scope('head'):
d['cls_head'] = tf.concat(
(d['flat_cls_head3'], d['flat_cls_head4'], d['flat_cls_head5'],
d['flat_cls_head6'], d['flat_cls_head7']),
axis=1)
d['loc_head'] = tf.concat(
(d['flat_loc_head3'], d['flat_loc_head4'], d['flat_loc_head5'],
d['flat_loc_head6'], d['flat_loc_head7']),
axis=1)
d['logits'] = tf.concat((d['loc_head'], d['cls_head']), axis=2)
d['pred'] = tf.concat(
(d['loc_head'], tf.nn.softmax(d['cls_head'], axis=-1)), axis=2)
return d
def _build_model(self, **kwargs):
"""
Build model.
:param kwargs: dict, extra arguments for building AlexNet.
- image_mean: np.ndarray, mean image for each input channel, shape: (C,).
- dropout_prob: float, the probability of dropping out each unit in FC layer.
:return d: dict, containing outputs on each layer.
"""
d = dict() # Dictionary to save intermediate values returned from each layer.
X_mean = kwargs.pop('image_mean', 0.0)
dropout_prob = kwargs.pop('dropout_prob', 0.0)
num_classes = int(self.y.get_shape()[-1])
# The probability of keeping each unit for dropout layers
keep_prob = tf.cond(self.is_train,
lambda: 1. - dropout_prob,
lambda: 1.)
# input
X_input = self.X - X_mean # perform mean subtraction
# First Convolution Layer
# conv1 - relu1 - pool1
with tf.variable_scope('conv1'):
# conv_layer(x, side_l, stride, out_depth, padding='SAME', **kwargs):
d['conv1'] = conv_layer(X_input, 3, 1, 64, padding='SAME',
weights_stddev=0.01, biases_value=1.0)
print('conv1.shape', d['conv1'].get_shape().as_list())
d['relu1'] = tf.nn.relu(d['conv1'])
# max_pool(x, side_l, stride, padding='SAME'):
d['pool1'] = max_pool(d['relu1'], 2, 1, padding='SAME')
d['drop1'] = tf.nn.dropout(d['pool1'], keep_prob)
print('pool1.shape', d['pool1'].get_shape().as_list())
# Second Convolution Layer
# conv2 - relu2 - pool2
with tf.variable_scope('conv2'):
d['conv2'] = conv_layer(d['pool1'], 3, 1, 128, padding='SAME',
weights_stddev=0.01, biases_value=1.0)
print('conv2.shape', d['conv2'].get_shape().as_list())
d['relu2'] = tf.nn.relu(d['conv2'])
d['pool2'] = max_pool(d['relu2'], 2, 1, padding='SAME')
d['drop2'] = tf.nn.dropout(d['pool2'], keep_prob)
print('pool2.shape', d['pool2'].get_shape().as_list())
# Third Convolution Layer
# conv3 - relu3
with tf.variable_scope('conv3'):
d['conv3'] = conv_layer(d['pool2'], 3, 1, 256, padding='SAME',
weights_stddev=0.01, biases_value=1.0)
print('conv3.shape', d['conv3'].get_shape().as_list())
d['relu3'] = tf.nn.relu(d['conv3'])
d['pool3'] = max_pool(d['relu3'], 2, 1, padding='SAME')
d['drop3'] = tf.nn.dropout(d['pool3'], keep_prob)
print('pool3.shape', d['pool3'].get_shape().as_list())
# Flatten feature maps
f_dim = int(np.prod(d['drop3'].get_shape()[1:]))
f_emb = tf.reshape(d['drop3'], [-1, f_dim])
# fc4
with tf.variable_scope('fc4'):
d['fc4'] = fc_layer(f_emb, 1024,
weights_stddev=0.005, biases_value=0.1)
d['relu4'] = tf.nn.relu(d['fc4'])
print('fc4.shape', d['relu4'].get_shape().as_list())
# fc5
with tf.variable_scope('fc5'):
d['fc5'] = fc_layer(d['relu4'], 1024,
weights_stddev=0.005, biases_value=0.1)
d['relu5'] = tf.nn.relu(d['fc5'])
print('fc5.shape', d['relu5'].get_shape().as_list())
d['logits'] = fc_layer(d['relu5'], num_classes,
weights_stddev=0.01, biases_value=0.0)
print('logits.shape', d['logits'].get_shape().as_list())
# softmax
d['pred'] = tf.nn.softmax(d['logits'])
return d
def _build_model(self, **kwargs):
"""
Build model.
:param kwargs: dict, extra arguments for building YOLO.
-image_mean: np.ndarray, mean image for each input channel, shape: (C,).
:return d: dict, containing outputs on each layer.
"""
d = dict()
x_mean = kwargs.pop('image_mean', 0.0)
# input
X_input = self.X - x_mean
is_train = self.is_train
#conv1 - batch_norm1 - leaky_relu1 - pool1
with tf.variable_scope('layer1'):
d['conv1'] = conv_layer(X_input,
3,
1,
32,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm1'] = batchNormalization(d['conv1'], is_train)
d['leaky_relu1'] = tf.nn.leaky_relu(d['batch_norm1'], alpha=0.1)
d['pool1'] = max_pool(d['leaky_relu1'], 2, 2, padding='SAME')
# (416, 416, 3) --> (208, 208, 32)
print('layer1.shape', d['pool1'].get_shape().as_list())
#conv2 - batch_norm2 - leaky_relu2 - pool2
with tf.variable_scope('layer2'):
d['conv2'] = depth_point_layer(d['pool1'],
3,
1,
64,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm2'] = batchNormalization(d['conv2'], is_train)
d['leaky_relu2'] = tf.nn.leaky_relu(d['batch_norm2'], alpha=0.1)
d['pool2'] = max_pool(d['leaky_relu2'], 2, 2, padding='SAME')
# (208, 208, 32) --> (104, 104, 64)
print('layer2.shape', d['pool2'].get_shape().as_list())
#conv3 - batch_norm3 - leaky_relu3
with tf.variable_scope('layer3'):
d['conv3'] = depth_point_layer(d['pool2'],
3,
1,
128,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm3'] = batchNormalization(d['conv3'], is_train)
d['leaky_relu3'] = tf.nn.leaky_relu(d['batch_norm3'], alpha=0.1)
# (104, 104, 64) --> (104, 104, 128)
print('layer3.shape', d['leaky_relu3'].get_shape().as_list())
#conv4 - batch_norm4 - leaky_relu4
with tf.variable_scope('layer4'):
d['conv4'] = conv_layer(d['leaky_relu3'],
1,
1,
64,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm4'] = batchNormalization(d['conv4'], is_train)
d['leaky_relu4'] = tf.nn.leaky_relu(d['batch_norm4'], alpha=0.1)
# (104, 104, 128) --> (104, 104, 64)
print('layer4.shape', d['leaky_relu4'].get_shape().as_list())
#conv5 - batch_norm5 - leaky_relu5 - pool5
with tf.variable_scope('layer5'):
d['conv5'] = depth_point_layer(d['leaky_relu4'],
3,
1,
128,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm5'] = batchNormalization(d['conv5'], is_train)
d['leaky_relu5'] = tf.nn.leaky_relu(d['batch_norm5'], alpha=0.1)
d['pool5'] = max_pool(d['leaky_relu5'], 2, 2, padding='SAME')
# (104, 104, 64) --> (52, 52, 128)
print('layer5.shape', d['pool5'].get_shape().as_list())
#conv6 - batch_norm6 - leaky_relu6
with tf.variable_scope('layer6'):
d['conv6'] = depth_point_layer(d['pool5'],
3,
1,
256,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm6'] = batchNormalization(d['conv6'], is_train)
d['leaky_relu6'] = tf.nn.leaky_relu(d['batch_norm6'], alpha=0.1)
# (52, 52, 128) --> (52, 52, 256)
print('layer6.shape', d['leaky_relu6'].get_shape().as_list())
#conv7 - batch_norm7 - leaky_relu7
with tf.variable_scope('layer7'):
d['conv7'] = conv_layer(d['leaky_relu6'],
1,
1,
128,
padding='SAME',
weights_stddev=0.01,
biases_value=0.0)
d['batch_norm7'] = batchNormalization(d['conv7'], is_train)
d['leaky_relu7'] = tf.nn.leaky_relu(d['batch_norm7'], alpha=0.1)
# (52, 52, 256) --> (52, 52, 128)
print('layer7.shape', d['leaky_relu7'].get_shape().as_list())
#conv8 - batch_norm8 - leaky_relu8 - pool8
with tf.variable_scope('layer8'):
d['conv8'] = depth_point_layer(d['leaky_relu7'],
3,
1,
256,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm8'] = batchNormalization(d['conv8'], is_train)
d['leaky_relu8'] = tf.nn.leaky_relu(d['batch_norm8'], alpha=0.1)
d['pool8'] = max_pool(d['leaky_relu8'], 2, 2, padding='SAME')
# (52, 52, 128) --> (26, 26, 256)
print('layer8.shape', d['pool8'].get_shape().as_list())
#conv9 - batch_norm9 - leaky_relu9
with tf.variable_scope('layer9'):
d['conv9'] = depth_point_layer(d['pool8'],
3,
1,
512,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm9'] = batchNormalization(d['conv9'], is_train)
d['leaky_relu9'] = tf.nn.leaky_relu(d['batch_norm9'], alpha=0.1)
# (26, 26, 256) --> (26, 26, 512)
print('layer9.shape', d['leaky_relu9'].get_shape().as_list())
#conv10 - batch_norm10 - leaky_relu10
with tf.variable_scope('layer10'):
d['conv10'] = conv_layer(d['leaky_relu9'],
1,
1,
256,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm10'] = batchNormalization(d['conv10'], is_train)
d['leaky_relu10'] = tf.nn.leaky_relu(d['batch_norm10'], alpha=0.1)
# (26, 26, 512) --> (26, 26, 256)
print('layer10.shape', d['leaky_relu10'].get_shape().as_list())
#conv11 - batch_norm11 - leaky_relu11
with tf.variable_scope('layer11'):
d['conv11'] = depth_point_layer(d['leaky_relu10'],
3,
1,
512,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm11'] = batchNormalization(d['conv11'], is_train)
d['leaky_relu11'] = tf.nn.leaky_relu(d['batch_norm11'], alpha=0.1)
# (26, 26, 256) --> (26, 26, 512)
print('layer11.shape', d['leaky_relu11'].get_shape().as_list())
#conv12 - batch_norm12 - leaky_relu12
with tf.variable_scope('layer12'):
d['conv12'] = conv_layer(d['leaky_relu11'],
1,
1,
256,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm12'] = batchNormalization(d['conv12'], is_train)
d['leaky_relu12'] = tf.nn.leaky_relu(d['batch_norm12'], alpha=0.1)
# (26, 26, 512) --> (26, 26, 256)
print('layer12.shape', d['leaky_relu12'].get_shape().as_list())
#conv13 - batch_norm13 - leaky_relu13 - pool13
with tf.variable_scope('layer13'):
d['conv13'] = depth_point_layer(d['leaky_relu12'],
3,
1,
512,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm13'] = batchNormalization(d['conv13'], is_train)
d['leaky_relu13'] = tf.nn.leaky_relu(d['batch_norm13'], alpha=0.1)
d['pool13'] = max_pool(d['leaky_relu13'], 2, 2, padding='SAME')
# (26, 26, 256) --> (13, 13, 512)
print('layer13.shape', d['pool13'].get_shape().as_list())
#conv14 - batch_norm14 - leaky_relu14
with tf.variable_scope('layer14'):
d['conv14'] = depth_point_layer(d['pool13'],
3,
1,
1024,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm14'] = batchNormalization(d['conv14'], is_train)
d['leaky_relu14'] = tf.nn.leaky_relu(d['batch_norm14'], alpha=0.1)
# (13, 13, 512) --> (13, 13, 1024)
print('layer14.shape', d['leaky_relu14'].get_shape().as_list())
#conv15 - batch_norm15 - leaky_relu15
with tf.variable_scope('layer15'):
d['conv15'] = conv_layer(d['leaky_relu14'],
1,
1,
512,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm15'] = batchNormalization(d['conv15'], is_train)
d['leaky_relu15'] = tf.nn.leaky_relu(d['batch_norm15'], alpha=0.1)
# (13, 13, 1024) --> (13, 13, 512)
print('layer15.shape', d['leaky_relu15'].get_shape().as_list())
#conv16 - batch_norm16 - leaky_relu16
with tf.variable_scope('layer16'):
d['conv16'] = depth_point_layer(d['leaky_relu15'],
3,
1,
1024,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm16'] = batchNormalization(d['conv16'], is_train)
d['leaky_relu16'] = tf.nn.leaky_relu(d['batch_norm16'], alpha=0.1)
# (13, 13, 512) --> (13, 13, 1024)
print('layer16.shape', d['leaky_relu16'].get_shape().as_list())
#conv17 - batch_norm16 - leaky_relu17
with tf.variable_scope('layer17'):
d['conv17'] = conv_layer(d['leaky_relu16'],
1,
1,
512,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm17'] = batchNormalization(d['conv17'], is_train)
d['leaky_relu17'] = tf.nn.leaky_relu(d['batch_norm17'], alpha=0.1)
# (13, 13, 1024) --> (13, 13, 512)
print('layer17.shape', d['leaky_relu17'].get_shape().as_list())
#conv18 - batch_norm18 - leaky_relu18
with tf.variable_scope('layer18'):
d['conv18'] = depth_point_layer(d['leaky_relu17'],
3,
1,
1024,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm18'] = batchNormalization(d['conv18'], is_train)
d['leaky_relu18'] = tf.nn.leaky_relu(d['batch_norm18'], alpha=0.1)
# (13, 13, 512) --> (13, 13, 1024)
print('layer18.shape', d['leaky_relu18'].get_shape().as_list())
#conv19 - batch_norm19 - leaky_relu19
with tf.variable_scope('layer19'):
d['conv19'] = depth_point_layer(d['leaky_relu18'],
3,
1,
1024,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm19'] = batchNormalization(d['conv19'], is_train)
d['leaky_relu19'] = tf.nn.leaky_relu(d['batch_norm19'], alpha=0.1)
# (13, 13, 1024) --> (13, 13, 1024)
print('layer19.shape', d['leaky_relu19'].get_shape().as_list())
#conv20 - batch_norm20 - leaky_relu20
with tf.variable_scope('layer20'):
d['conv20'] = depth_point_layer(d['leaky_relu19'],
3,
1,
1024,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm20'] = batchNormalization(d['conv20'], is_train)
d['leaky_relu20'] = tf.nn.leaky_relu(d['batch_norm20'], alpha=0.1)
# (13, 13, 1024) --> (13, 13, 1024)
print('layer20.shape', d['leaky_relu20'].get_shape().as_list())
# concatenate layer20 and layer 13 using space to depth
with tf.variable_scope('layer21'):
d['skip_connection'] = conv_layer(d['leaky_relu13'],
1,
1,
64,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['skip_batch'] = batchNormalization(d['skip_connection'],
is_train)
d['skip_leaky_relu'] = tf.nn.leaky_relu(d['skip_batch'], alpha=0.1)
d['skip_space_to_depth_x2'] = tf.space_to_depth(
d['skip_leaky_relu'], block_size=2)
d['concat21'] = tf.concat(
[d['skip_space_to_depth_x2'], d['leaky_relu20']], axis=-1)
# (13, 13, 1024) --> (13, 13, 256+1024)
print('layer21.shape', d['concat21'].get_shape().as_list())
#conv22 - batch_norm22 - leaky_relu22
with tf.variable_scope('layer22'):
d['conv22'] = depth_point_layer(d['concat21'],
3,
1,
1024,
padding='SAME',
use_bias=False,
weights_stddev=0.01)
d['batch_norm22'] = batchNormalization(d['conv22'], is_train)
d['leaky_relu22'] = tf.nn.leaky_relu(d['batch_norm22'], alpha=0.1)
# (13, 13, 1280) --> (13, 13, 1024)
print('layer22.shape', d['leaky_relu22'].get_shape().as_list())
output_channel = self.num_anchors * (5 + self.num_classes)
d['logit'] = conv_layer(d['leaky_relu22'],
1,
1,
output_channel,
padding='SAME',
use_bias=True,
weights_stddev=0.01,
biases_value=0.1)
d['pred'] = tf.reshape(d['logit'],
(-1, self.grid_size[0], self.grid_size[1],
self.num_anchors, 5 + self.num_classes))
print('pred.shape', d['pred'].get_shape().as_list())
# (13, 13, 1024) --> (13, 13, num_anchors , (5 + num_classes))
return d
def _build_model(self, **kwargs):
"""
Build model.
:param kwargs: dict, extra arguments for building YOLO.
-image_mean: np.ndarray, mean image for each input channel, shape: (C,).
:return d: dict, containing outputs on each layer.
"""
d = dict()
x_mean = kwargs.pop('image_mean', 0.0)
pretrain = kwargs.pop('pretrain', False)
frontend = kwargs.pop('frontend', 'resnet_v2_50')
# input
X_input = self.X - x_mean
is_train = self.is_train
# Feature Extractor
if pretrain:
frontend_dir = os.path.join('pretrained_models',
'{}.ckpt'.format(frontend))
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits, end_points = resnet_v2.resnet_v2_50(
self.X, is_training=self.is_train)
d['init_fn'] = slim.assign_from_checkpoint_fn(
model_path=frontend_dir,
var_list=slim.get_model_variables(frontend))
convs = [
end_points[frontend + '/block{}'.format(x)] for x in [4, 2, 1]
]
d['conv_s32'] = convs[0]
d['conv_s16'] = convs[1]
else:
# Build ConvNet
#conv1 - batch_norm1 - leaky_relu1 - pool1
with tf.variable_scope('layer1'):
d['conv1'] = conv_bn_relu(X_input, 32, (3, 3), is_train)
d['pool1'] = max_pool(d['conv1'], 2, 2, padding='SAME')
# (416, 416, 3) --> (208, 208, 32)
#conv2 - batch_norm2 - leaky_relu2 - pool2
with tf.variable_scope('layer2'):
d['conv2'] = conv_bn_relu(d['pool1'], 64, (3, 3), is_train)
d['pool2'] = max_pool(d['conv2'], 2, 2, padding='SAME')
# (208, 208, 32) --> (104, 104, 64)
#conv3 - batch_norm3 - leaky_relu3
with tf.variable_scope('layer3'):
d['conv3'] = conv_bn_relu(d['pool2'], 128, (3, 3), is_train)
# (104, 104, 64) --> (104, 104, 128)
#conv4 - batch_norm4 - leaky_relu4
with tf.variable_scope('layer4'):
d['conv4'] = conv_bn_relu(d['conv3'], 64, (1, 1), is_train)
# (104, 104, 128) --> (104, 104, 64)
#conv5 - batch_norm5 - leaky_relu5 - pool5
with tf.variable_scope('layer5'):
d['conv5'] = conv_bn_relu(d['conv4'], 128, (3, 3), is_train)
d['pool5'] = max_pool(d['conv5'], 2, 2, padding='SAME')
# (104, 104, 64) --> (52, 52, 128)
#conv6 - batch_norm6 - leaky_relu6
with tf.variable_scope('layer6'):
d['conv6'] = conv_bn_relu(d['pool5'], 256, (3, 3), is_train)
# (52, 52, 128) --> (52, 52, 256)
#conv7 - batch_norm7 - leaky_relu7
with tf.variable_scope('layer7'):
d['conv7'] = conv_bn_relu(d['conv6'], 128, (1, 1), is_train)
# (52, 52, 256) --> (52, 52, 128)
#conv8 - batch_norm8 - leaky_relu8 - pool8
with tf.variable_scope('layer8'):
d['conv8'] = conv_bn_relu(d['conv7'], 256, (3, 3), is_train)
d['pool8'] = max_pool(d['conv8'], 2, 2, padding='SAME')
# (52, 52, 128) --> (26, 26, 256)
#conv9 - batch_norm9 - leaky_relu9
with tf.variable_scope('layer9'):
d['conv9'] = conv_bn_relu(d['pool8'], 512, (3, 3), is_train)
# (26, 26, 256) --> (26, 26, 512)
#conv10 - batch_norm10 - leaky_relu10
with tf.variable_scope('layer10'):
d['conv10'] = conv_bn_relu(d['conv9'], 256, (1, 1), is_train)
# (26, 26, 512) --> (26, 26, 256)
#conv11 - batch_norm11 - leaky_relu11
with tf.variable_scope('layer11'):
d['conv11'] = conv_bn_relu(d['conv10'], 512, (3, 3), is_train)
# (26, 26, 256) --> (26, 26, 512)
#conv12 - batch_norm12 - leaky_relu12
with tf.variable_scope('layer12'):
d['conv12'] = conv_bn_relu(d['conv11'], 256, (1, 1), is_train)
# (26, 26, 512) --> (26, 26, 256)
#conv13 - batch_norm13 - leaky_relu13 - pool13
with tf.variable_scope('layer13'):
d['conv13'] = conv_bn_relu(d['conv12'], 512, (3, 3), is_train)
d['pool13'] = max_pool(d['conv13'], 2, 2, padding='SAME')
# (26, 26, 256) --> (13, 13, 512)
#conv14 - batch_norm14 - leaky_relu14
with tf.variable_scope('layer14'):
d['conv14'] = conv_bn_relu(d['pool13'], 1024, (3, 3), is_train)
# (13, 13, 512) --> (13, 13, 1024)
#conv15 - batch_norm15 - leaky_relu15
with tf.variable_scope('layer15'):
d['conv15'] = conv_bn_relu(d['conv14'], 512, (1, 1), is_train)
# (13, 13, 1024) --> (13, 13, 512)
#conv16 - batch_norm16 - leaky_relu16
with tf.variable_scope('layer16'):
d['conv16'] = conv_bn_relu(d['conv15'], 1024, (3, 3), is_train)
# (13, 13, 512) --> (13, 13, 1024)
#conv17 - batch_norm16 - leaky_relu17
with tf.variable_scope('layer17'):
d['conv17'] = conv_bn_relu(d['conv16'], 512, (1, 1), is_train)
# (13, 13, 1024) --> (13, 13, 512)
#conv18 - batch_norm18 - leaky_relu18
with tf.variable_scope('layer18'):
d['conv18'] = conv_bn_relu(d['conv17'], 1024, (3, 3), is_train)
# (13, 13, 512) --> (13, 13, 1024)
#conv19 - batch_norm19 - leaky_relu19
with tf.variable_scope('layer19'):
d['conv19'] = conv_bn_relu(d['conv18'], 1024, (3, 3), is_train)
# (13, 13, 1024) --> (13, 13, 1024)
d['conv_s32'] = d['conv19']
d['conv_s16'] = d['conv13']
#Detection Layer
#conv20 - batch_norm20 - leaky_relu20
with tf.variable_scope('layer20'):
d['conv20'] = conv_bn_relu(d['conv_s32'], 1024, (3, 3), is_train)
# (13, 13, 1024) --> (13, 13, 1024)
# concatenate layer20 and layer 13 using space to depth
with tf.variable_scope('layer21'):
d['skip_connection'] = conv_bn_relu(d['conv_s16'], 64, (1, 1),
is_train)
d['skip_space_to_depth_x2'] = tf.space_to_depth(
d['skip_connection'], block_size=2)
d['concat21'] = tf.concat(
[d['skip_space_to_depth_x2'], d['conv20']], axis=-1)
# (13, 13, 1024) --> (13, 13, 256+1024)
#conv22 - batch_norm22 - leaky_relu22
with tf.variable_scope('layer22'):
d['conv22'] = conv_bn_relu(d['concat21'], 1024, (3, 3), is_train)
# (13, 13, 1280) --> (13, 13, 1024)
output_channel = self.num_anchors * (5 + self.num_classes)
d['logits'] = conv_layer(d['conv22'],
output_channel, (1, 1), (1, 1),
padding='SAME',
use_bias=True)
d['pred'] = tf.reshape(d['logits'],
(-1, self.grid_size[0], self.grid_size[1],
self.num_anchors, 5 + self.num_classes))
# (13, 13, 1024) --> (13, 13, num_anchors , (5 + num_classes))
return d
def _build_model(self, **kwargs):
d = dict()
num_classes = self.num_classes
pretrain = kwargs.pop('pretrain', True)
frontend = kwargs.pop('frontend', 'resnet_v2_50')
if pretrain:
frontend_dir = os.path.join('pretrained_models',
'{}.ckpt'.format(frontend))
print(frontend_dir)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits, end_points = resnet_v2.resnet_v2_50(
self.X, is_training=self.is_train)
d['init_fn'] = slim.assign_from_checkpoint_fn(
model_path=frontend_dir,
var_list=slim.get_model_variables(frontend))
resnet_dict = [
'/block1/unit_2/bottleneck_v2', # conv1
'/block2/unit_3/bottleneck_v2', # conv2
'/block3/unit_5/bottleneck_v2', # conv3
'/block4/unit_3/bottleneck_v2' # conv4
]
convs = [end_points[frontend + x] for x in resnet_dict]
else:
# TODO build convNet
raise NotImplementedError("Build own convNet!")
if self.X.shape[1].value is None:
# input size should be bigger than (512, 512)
g_kernel_size = (15, 15)
else:
g_kernel_size = (self.X.shape[1].value // 32 - 1,
self.X.shape[2].value // 32 - 1)
with tf.variable_scope('layer5'):
d['gcm1'] = global_conv_module(convs[3], num_classes,
g_kernel_size)
d['brm1_1'] = boundary_refine_module(d['gcm1'], num_classes)
d['up16'] = up_scale(d['brm1_1'], 2)
with tf.variable_scope('layer4'):
d['gcm2'] = global_conv_module(convs[2], num_classes,
g_kernel_size)
d['brm2_1'] = boundary_refine_module(d['gcm2'], num_classes)
d['sum16'] = d['up16'] + d['brm2_1']
d['brm2_2'] = boundary_refine_module(d['sum16'], num_classes)
d['up8'] = up_scale(d['brm2_2'], 2)
with tf.variable_scope('layer3'):
d['gcm3'] = global_conv_module(convs[1], num_classes,
g_kernel_size)
d['brm3_1'] = boundary_refine_module(d['gcm3'], num_classes)
d['sum8'] = d['up8'] + d['brm3_1']
d['brm3_2'] = boundary_refine_module(d['sum8'], num_classes)
d['up4'] = up_scale(d['brm3_2'], 2)
with tf.variable_scope('layer2'):
d['gcm4'] = global_conv_module(convs[0], num_classes,
g_kernel_size)
d['brm4_1'] = boundary_refine_module(d['gcm4'], num_classes)
d['sum4'] = d['up4'] + d['brm4_1']
d['brm4_2'] = boundary_refine_module(d['sum4'], num_classes)
d['up2'] = up_scale(d['brm4_2'], 2)
with tf.variable_scope('layer1'):
d['brm4_3'] = boundary_refine_module(d['up2'], num_classes)
d['up1'] = up_scale(d['brm4_3'], 2)
d['brm4_4'] = boundary_refine_module(d['up1'], num_classes)
with tf.variable_scope('output_layer'):
d['logits'] = conv_layer(d['brm4_4'], num_classes, (1, 1), (1, 1))
d['pred'] = tf.nn.softmax(d['logits'], axis=-1)
return d
def _build_model(self, **kwargs):
"""
Build model.
:param kwargs: dict, extra arguments for building AlexNet.
- image_mean: np.ndarray, mean image for each input channel, shape: (C,).
- dropout_prob: float, the probability of dropping out each unit in FC layer.
:return d: dict, containing outputs on each layer.
"""
d = dict(
) # Dictionary to save intermediate values returned from each layer.
X_mean = kwargs.pop('image_mean', 0.0)
dropout_prob = kwargs.pop('dropout_prob', 0.0)
num_classes = int(self.y.get_shape()[-1])
# The probability of keeping each unit for dropout layers
keep_prob = tf.cond(self.is_train, lambda: 1. - dropout_prob,
lambda: 1.)
# input
X_input = self.X - X_mean # perform mean subtraction
# conv1 - relu1 - pool1
with tf.variable_scope('conv1'):
d['conv1'] = conv_layer(X_input,
11,
4,
96,
padding='VALID',
weights_stddev=0.01,
biases_value=0.0)
print('conv1.shape', d['conv1'].get_shape().as_list())
d['relu1'] = tf.nn.relu(d['conv1'])
# (227, 227, 3) --> (55, 55, 96)
d['pool1'] = max_pool(d['relu1'], 3, 2, padding='VALID')
# (55, 55, 96) --> (27, 27, 96)
print('pool1.shape', d['pool1'].get_shape().as_list())
# conv2 - relu2 - pool2
with tf.variable_scope('conv2'):
d['conv2'] = conv_layer(d['pool1'],
5,
1,
256,
padding='SAME',
weights_stddev=0.01,
biases_value=0.1)
print('conv2.shape', d['conv2'].get_shape().as_list())
d['relu2'] = tf.nn.relu(d['conv2'])
# (27, 27, 96) --> (27, 27, 256)
d['pool2'] = max_pool(d['relu2'], 3, 2, padding='VALID')
# (27, 27, 256) --> (13, 13, 256)
print('pool2.shape', d['pool2'].get_shape().as_list())
# conv3 - relu3
with tf.variable_scope('conv3'):
d['conv3'] = conv_layer(d['pool2'],
3,
1,
384,
padding='SAME',
weights_stddev=0.01,
biases_value=0.0)
print('conv3.shape', d['conv3'].get_shape().as_list())
d['relu3'] = tf.nn.relu(d['conv3'])
# (13, 13, 256) --> (13, 13, 384)
# conv4 - relu4
with tf.variable_scope('conv4'):
d['conv4'] = conv_layer(d['relu3'],
3,
1,
384,
padding='SAME',
weights_stddev=0.01,
biases_value=0.1)
print('conv4.shape', d['conv4'].get_shape().as_list())
d['relu4'] = tf.nn.relu(d['conv4'])
# (13, 13, 384) --> (13, 13, 384)
# conv5 - relu5 - pool5
with tf.variable_scope('conv5'):
d['conv5'] = conv_layer(d['relu4'],
3,
1,
256,
padding='SAME',
weights_stddev=0.01,
biases_value=0.1)
print('conv5.shape', d['conv5'].get_shape().as_list())
d['relu5'] = tf.nn.relu(d['conv5'])
# (13, 13, 384) --> (13, 13, 256)
d['pool5'] = max_pool(d['relu5'], 3, 2, padding='VALID')
# (13, 13, 256) --> (6, 6, 256)
print('pool5.shape', d['pool5'].get_shape().as_list())
# Flatten feature maps
f_dim = int(np.prod(d['pool5'].get_shape()[1:]))
f_emb = tf.reshape(d['pool5'], [-1, f_dim])
# (6, 6, 256) --> (9216)
# fc6
with tf.variable_scope('fc6'):
d['fc6'] = fc_layer(f_emb,
4096,
weights_stddev=0.005,
biases_value=0.1)
d['relu6'] = tf.nn.relu(d['fc6'])
d['drop6'] = tf.nn.dropout(d['relu6'], keep_prob)
# (9216) --> (4096)
print('drop6.shape', d['drop6'].get_shape().as_list())
# fc7
with tf.variable_scope('fc7'):
d['fc7'] = fc_layer(d['drop6'],
4096,
weights_stddev=0.005,
biases_value=0.1)
d['relu7'] = tf.nn.relu(d['fc7'])
d['drop7'] = tf.nn.dropout(d['relu7'], keep_prob)
# (4096) --> (4096)
print('drop7.shape', d['drop7'].get_shape().as_list())
# fc8
with tf.variable_scope('fc8'):
d['logits'] = fc_layer(d['relu7'],
num_classes,
weights_stddev=0.01,
biases_value=0.0)
# (4096) --> (num_classes)
# softmax
d['pred'] = tf.nn.softmax(d['logits'])
return d
def cnn_model_trainer():
# ALEXNET
dataset = DeepSatData()
x = tf.placeholder(tf.float32, shape=[None, 28, 28, 4], name='x')
y_ = tf.placeholder(tf.float32, shape=[None, 4], name='y_')
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
conv1 = conv_layer(x, shape=[3, 3, 4, 16], pad='VALID')
conv1_pool = max_pool_2x2(conv1, 2, 2)
conv2 = conv_layer(conv1_pool, shape=[3, 3, 16, 48], pad='SAME')
conv2_pool = max_pool_2x2(conv2, 3, 3)
conv3 = conv_layer(conv2_pool, shape=[3, 3, 48, 96], pad='SAME')
# conv3_pool = max_pool_2x2(conv3)
conv4 = conv_layer(conv3, shape=[3, 3, 96, 64], pad='SAME')
# conv4_pool = max_pool_2x2(conv4)
conv5 = conv_layer(conv4, shape=[3, 3, 64, 64], pad='SAME')
conv5_pool = max_pool_2x2(conv5, 2, 2)
_flat = tf.reshape(conv5_pool, [-1, 3 * 3 * 64])
_drop1 = tf.nn.dropout(_flat, keep_prob=keep_prob)
# full_1 = tf.nn.relu(full_layer(_drop1, 200))
full_1 = tf.nn.relu(full_layer(_drop1, 200))
# -- until here
# classifier:add(nn.Threshold(0, 1e-6))
_drop2 = tf.nn.dropout(full_1, keep_prob=keep_prob)
full_2 = tf.nn.relu(full_layer(_drop2, 200))
# classifier:add(nn.Threshold(0, 1e-6))
full_3 = full_layer(full_2, 4)
pred = tf.nn.softmax(logits=full_3, name='pred') # for later prediction
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=full_3, labels=y_))
# train_step = tf.train.RMSPropOptimizer(lr, decay, momentum).minimize(cross_entropy)
train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(full_3, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')
tf.summary.scalar('loss', cross_entropy)
tf.summary.scalar('accuracy', accuracy)
# Setting up for the visualization of the data in Tensorboard
embedding_size = 200 # size of second to last fc layer
embedding_input = full_2 #FC2 as input
# Variable containing the points in visualization
embedding = tf.Variable(tf.zeros([10000, embedding_size]), name="test_embedding")
assignment = embedding.assign(embedding_input) # Will be passed in the test session
merged_sum = tf.summary.merge_all()
def test(test_sess, assign):
x_ = dataset.test.images.reshape(10, 10000, 28, 28, 4)
y = dataset.test.labels.reshape(10, 10000, 4)
test_acc = np.mean([test_sess.run(accuracy, feed_dict={x: x_[im], y_: y[im], keep_prob: 1.0})
for im in range(10)])
# Pass through the last 10,000 of the test set for visualization
test_sess.run([assign], feed_dict={x: x_[9], y_: y[9], keep_prob: 1.0})
return test_acc
# config=config
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# tensorboard
sum_writer = tf.summary.FileWriter(os.path.join(log_dir, log_name))
sum_writer.add_graph(sess.graph)
# Create a Saver object
#max_to_keep: keep how many models to keep. Delete old ones.
saver = tf.train.Saver(max_to_keep=MODELS_TO_KEEP)
# setting up Projector
config = tf.contrib.tensorboard.plugins.projector.ProjectorConfig()
embedding_config = config.embeddings.add()
embedding_config.tensor_name = embedding.name
embedding_config.metadata_path = LABELS #labels
# Specify the width and height of a single thumbnail.
embedding_config.sprite.image_path = SPRITES
embedding_config.sprite.single_image_dim.extend([28, 28])
tf.contrib.tensorboard.plugins.projector.visualize_embeddings(sum_writer, config)
for i in range(STEPS):
batch = dataset.train.next_batch(BATCH_SIZE)
batch_x = batch[0]
batch_y = batch[1]
sess.run(train_step, feed_dict={x: batch_x, y_: batch_y, keep_prob: dropoutProb})
_, summ = sess.run([train_step, merged_sum], feed_dict={x: batch_x, y_: batch_y, keep_prob: dropoutProb})
sum_writer.add_summary(summ, i)
if i % ONE_EPOCH == 0:
ep_print = "\n*****************EPOCH: %d" % ((i/ONE_EPOCH) + 1)
write_to_file.write(ep_print)
print(ep_print)
if i % TEST_INTERVAL == 0:
acc = test(sess, assignment)
loss = sess.run(cross_entropy, feed_dict={x: batch_x, y_: batch_y, keep_prob: dropoutProb})
ep_test_print = "\nEPOCH:%d" % ((i/ONE_EPOCH) + 1) + " Step:" + str(i) + \
"|| Minibatch Loss= " + "{:.4f}".format(loss) + \
" Accuracy: {:.4}%".format(acc * 100)
write_to_file.write(ep_test_print)
print(ep_test_print)
# Create a checkpoint in every iteration
# saver.save(sess, os.path.join(model_dir, model_name),
# global_step=i)
saver.save(sess, os.path.join(model_dir, 'model.ckpt'), global_step=i)
test(sess, assignment)
sum_writer.close()
def run_simple_net():
# mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
dataset = DeepSatData()
x = tf.placeholder(tf.float32, shape=[None, 28, 28, 4])
y_ = tf.placeholder(tf.float32, shape=[None, 4])
keep_prob = tf.placeholder(tf.float32)
conv1 = conv_layer(x, shape=[3, 3, 4, 16], pad='VALID')
conv1_pool = max_pool_2x2(conv1, 2, 2)
conv2 = conv_layer(conv1_pool, shape=[3, 3, 16, 48], pad='SAME')
conv2_pool = max_pool_2x2(conv2, 3, 3)
conv3 = conv_layer(conv2_pool, shape=[3, 3, 48, 96], pad='SAME')
# conv3_pool = max_pool_2x2(conv3)
conv4 = conv_layer(conv3, shape=[3, 3, 96, 64], pad='SAME')
# conv4_pool = max_pool_2x2(conv4)
conv5 = conv_layer(conv4, shape=[3, 3, 64, 64], pad='SAME')
conv5_pool = max_pool_2x2(conv5, 2, 2)
_flat = tf.reshape(conv5_pool, [-1, 3 * 3 * 64])
_drop1 = tf.nn.dropout(_flat, keep_prob=keep_prob)
# full_1 = tf.nn.relu(full_layer(_drop1, 200))
full_1 = tf.nn.relu(full_layer(_drop1, 200))
# -- until here
# classifier:add(nn.Threshold(0, 1e-6))
_drop2 = tf.nn.dropout(full_1, keep_prob=keep_prob)
full_2 = tf.nn.relu(full_layer(_drop2, 200))
# classifier:add(nn.Threshold(0, 1e-6))
full_3 = full_layer(full_2, 4)
predict = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=full_3, labels=y_))
train_step = tf.train.RMSPropOptimizer(lr, decay,
momentum).minimize(predict)
correct_prediction = tf.equal(tf.argmax(full_3, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# full1_drop = tf.nn.dropout(full_1, keep_prob=keep_prob)
# y_conv = full_layer(full1_drop, 10)
#
# cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv,
# labels=y_))
#
# train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)
#
# correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
# accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#
tf.summary.scalar('loss', predict)
tf.summary.scalar('accuracy', accuracy)
merged_sum = tf.summary.merge_all()
def test(sess):
X = dataset.test.images.reshape(10, 10000, 28, 28, 4)
Y = dataset.test.labels.reshape(10, 10000, 4)
acc = np.mean([
sess.run(accuracy, feed_dict={
x: X[i],
y_: Y[i],
keep_prob: 1.0
}) for i in range(10)
])
print("Accuracy: {:.4}%".format(acc * 100))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sum_writer = tf.summary.FileWriter('logs/' + 'default')
sum_writer.add_graph(sess.graph)
for i in range(STEPS):
batch = dataset.train.next_batch(BATCH_SIZE)
batch_x = batch[0]
batch_y = batch[1]
_, summ = sess.run([train_step, merged_sum],
feed_dict={
x: batch_x,
y_: batch_y,
keep_prob: 0.5
})
sum_writer.add_summary(summ, i)
sess.run(train_step,
feed_dict={
x: batch_x,
y_: batch_y,
keep_prob: 0.5
})
if i % 500 == 0:
test(sess)
test(sess)
sum_writer.close()
