def train(loss_val, var_list):
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
grads = optimizer.compute_gradients(loss_val, var_list=var_list)
if FLAGS.debug:
# print(len(var_list))
for grad, var in grads:
utils.add_gradient_summary(grad, var)
return optimizer.apply_gradients(grads)
def train(loss_val, var_list):
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
grads = optimizer.compute_gradients(loss_val, var_list=var_list)
if FLAGS.debug:
# print(len(var_list))
for grad, var in grads:
utils.add_gradient_summary(grad, var)
return optimizer.apply_gradients(grads)
def inference(image):
weights_path = os.getcwd() + "\\models\\" + "new_mob718"
weights = torch.load(weights_path)
with tf.variable_scope("inference"):
image_net = dlv3p_718(weights, image)
low_level_feat = image_net['low_features']
high_level_feat = image_net['high_features']
high_shape = high_level_feat.get_shape().as_list()
x_aspp = utils.aspp_dl(high_level_feat, 320, 256)
x_ = utils.global_avgp_dl(high_level_feat,
320,
256,
name='global_avgp')
x_ = utils.upsample_dl(x_, high_shape[1], high_shape[2])
high_level_feat = tf.concat([x_aspp, x_], 3,
name="fuse_oct_1") # 1/8 feature maps
high_level_feat = utils.aspp_conv2d_dl(high_level_feat,
1280,
256,
1,
name='conv_h')
high_level_feat = utils.aspp_bn_dl(high_level_feat, name="bn_h")
high_shape = high_level_feat.get_shape().as_list()
high_level_feat = utils.upsample_dl(
high_level_feat, 2 * (high_shape[1]),
2 * (high_shape[2])) # 1/4 feature_maps
low_level_feat = utils.aspp_conv2d_dl(low_level_feat,
24,
48,
1,
name="conv_l")
low_level_feat = utils.aspp_bn_dl(low_level_feat, name="bn_l")
high_level_feat = tf.concat([high_level_feat, low_level_feat], 3)
high_level_feat = utils.aspp_conv2d_dl(high_level_feat,
304,
20,
1,
name="conv_pred")
high_shape = high_level_feat.get_shape().as_list()
high_level_feat = utils.upsample_dl(high_level_feat, 4 * high_shape[1],
4 * high_shape[2])
annotation_pred = tf.argmax(high_level_feat,
dimension=3,
name='prediction')
# annotation_pred = high_level_feat
return tf.expand_dims(annotation_pred, dim=3), high_level_feat
def dense_de_block(weights, feature_maps, block_idx, block_outputs):
with tf.variable_scope("denseblock%d" % block_idx):
global output
tran_input = dense_de_layer(weights, feature_maps, block_idx)
if (block_idx < 4):
output = utils.tran_de(weights, tran_input, "tran%d" % block_idx)
block_outputs.append(output)
if (block_idx == 4):
block_outputs.append(tran_input)
#return output, block_outputs
return block_outputs
def vgg_net(weights, image):
layers = ('conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1', 'conv2_1',
'relu2_1', 'conv2_2', 'relu2_2', 'pool2', 'conv3_1', 'relu3_1',
'conv3_2', 'relu3_2', 'conv3_3', 'relu3_3', 'conv3_4', 'relu3_4',
'pool3', 'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3',
'relu4_3', 'conv4_4', 'relu4_4', 'pool4', 'conv5_1', 'relu5_1',
'conv5_2', 'relu5_2', 'conv5_3', 'relu5_3', 'conv5_4', 'relu5_4')
net = {}
current = image
for i, name in enumerate(layers):
if name in [
'conv3_4', 'relu3_4', 'conv4_4', 'relu4_4', 'conv5_4',
'relu5_4'
]:
continue
kind = name[:4]
if kind == 'conv':
kernels, bias = weights[i][0][0][0][0]
# matconvnet: weights are [width, height, in_channels, out_channels]
# tensorflow: weights are [height, width, in_channels, out_channels]
kernels = utils.get_variable(np.transpose(kernels, (1, 0, 2, 3)),
name=name + "_w")
bias = utils.get_variable(bias.reshape(-1), name=name + "_b")
current = utils.conv2d_basic(current, kernels, bias)
elif kind == 'relu':
current = tf.nn.relu(current, name=name)
if FLAGS.debug:
utils.add_activation_summary(current)
elif kind == 'pool':
current = utils.avg_pool_2x2(current)
net[name] = current
return net
def dense_de_sub_layer(weights, feature_maps, block_idx, layer_idx):
with tf.variable_scope("denselayer%d" % layer_idx):
l_scope = features + denseblock + (
str)(block_idx) + '/' + denselayer + (str)(layer_idx) + '/'
nk = get_nk(weights, block_idx, layer_idx)
c1k = weights[l_scope + conv1 + sub_l[0]]
c2k = weights[l_scope + conv2 + sub_l[0]]
res = utils.batch_norm_de(feature_maps,
nk[2],
nk[3],
nk[1],
nk[0],
scope='bn1')
res = utils.relu_de(res, name='relu1')
res = utils.conv2d_de(res, c1k, 1, 1, name='conv1')
res = utils.batch_norm_de(res, nk[6], nk[7], nk[5], nk[4], scope='bn2')
res = utils.relu_de(res, name='relu2')
res = utils.conv2d_de(res, c2k, 3, 1, name='conv2')
return res
def vgg_net(weights, image):
layers = (
'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1',
'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2',
'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'conv3_3',
'relu3_3', 'conv3_4', 'relu3_4', 'pool3',
'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3',
'relu4_3', 'conv4_4', 'relu4_4', 'pool4',
'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'conv5_3',
'relu5_3', 'conv5_4', 'relu5_4'
)
net = {}
current = image
for i, name in enumerate(layers):
if name in ['conv3_4', 'relu3_4', 'conv4_4', 'relu4_4', 'conv5_4', 'relu5_4']:
continue
kind = name[:4]
if kind == 'conv':
kernels, bias = weights[i][0][0][0][0]
# matconvnet: weights are [width, height, in_channels, out_channels]
# tensorflow: weights are [height, width, in_channels, out_channels]
kernels = utils.get_variable(np.transpose(kernels, (1, 0, 2, 3)), name=name + "_w")
bias = utils.get_variable(bias.reshape(-1), name=name + "_b")
current = utils.conv2d_basic(current, kernels, bias)
elif kind == 'relu':
current = tf.nn.relu(current, name=name)
if FLAGS.debug:
utils.add_activation_summary(current)
elif kind == 'pool':
current = utils.avg_pool_2x2(current)
net[name] = current
return net
def inference(image, keep_prob):
"""
Semantic segmentation network definition
:param image: input image. Should have values in range 0-255
:param keep_prob:
:return:
"""
print("setting up vgg initialized conv layers ...")
model_data = utils.get_model_data(FLAGS.model_dir, MODEL_URL)
mean = model_data['normalization'][0][0][0]
mean_pixel = np.mean(mean, axis=(0, 1))
weights = np.squeeze(model_data['layers'])
#processed_image = utils.process_image(image, mean_pixel)
with tf.variable_scope("inference"):
image_net = vgg_net(weights, image)
conv_final_layer = image_net["conv5_3"]
pool5 = utils.max_pool_2x2(conv_final_layer)
W6 = utils.weight_variable([7, 7, 512, 4096], name="W6")
b6 = utils.bias_variable([4096], name="b6")
conv6 = utils.conv2d_basic(pool5, W6, b6)
relu6 = tf.nn.relu(conv6, name="relu6")
if FLAGS.debug:
utils.add_activation_summary(relu6)
relu_dropout6 = tf.nn.dropout(relu6, keep_prob=keep_prob)
W7 = utils.weight_variable([1, 1, 4096, 4096], name="W7")
b7 = utils.bias_variable([4096], name="b7")
conv7 = utils.conv2d_basic(relu_dropout6, W7, b7)
relu7 = tf.nn.relu(conv7, name="relu7")
if FLAGS.debug:
utils.add_activation_summary(relu7)
relu_dropout7 = tf.nn.dropout(relu7, keep_prob=keep_prob)
W8 = utils.weight_variable([1, 1, 4096, NUM_OF_CLASSESS], name="W8")
b8 = utils.bias_variable([NUM_OF_CLASSESS], name="b8")
conv8 = utils.conv2d_basic(relu_dropout7, W8, b8)
# annotation_pred1 = tf.argmax(conv8, axis=3, name="prediction1")
# now to upscale to actual image size
deconv_shape1 = image_net["pool4"].get_shape()
W_t1 = utils.weight_variable(
[4, 4, deconv_shape1[3].value, NUM_OF_CLASSESS], name="W_t1")
b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1")
conv_t1 = utils.conv2d_transpose_strided(conv8,
W_t1,
b_t1,
output_shape=tf.shape(
image_net["pool4"]))
fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1")
deconv_shape2 = image_net["pool3"].get_shape()
W_t2 = utils.weight_variable(
[4, 4, deconv_shape2[3].value, deconv_shape1[3].value],
name="W_t2")
b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2")
conv_t2 = utils.conv2d_transpose_strided(fuse_1,
W_t2,
b_t2,
output_shape=tf.shape(
image_net["pool3"]))
fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2")
shape = tf.shape(image)
deconv_shape3 = tf.stack(
[shape[0], shape[1], shape[2], NUM_OF_CLASSESS])
W_t3 = utils.weight_variable(
[16, 16, NUM_OF_CLASSESS, deconv_shape2[3].value], name="W_t3")
b_t3 = utils.bias_variable([NUM_OF_CLASSESS], name="b_t3")
conv_t3 = utils.conv2d_transpose_strided(fuse_2,
W_t3,
b_t3,
output_shape=deconv_shape3,
stride=8)
annotation_pred = tf.argmax(conv_t3, axis=3, name="prediction")
return tf.expand_dims(annotation_pred, dim=3), conv_t3
def dense_de(weights, refine_weights, image):
# define and hold the char for the dense_de model
block_outputs = []
current = image
nk = [] # norm kernel list
## adding for resize the image
## org_shape = image.get_shape().as_list()
## _, org_height, org_width, channels = org_shape
with tf.variable_scope("inference"):
# add structure into the model
# first layer(s)
kernels = weights[features + conv0 + sub_l[0]]
current = utils.conv2d_de(current, kernels, 7, stride=2, name='conv0')
nk = get_nk(weights, 0, 0)
current = utils.batch_norm_de(current,
nk[2],
nk[3],
nk[1],
nk[0],
scope='bn0')
current = utils.relu_de(current, name='relu0')
current = utils.maxpool_de(current,
pool_size=3,
stride=2,
name='pool0')
block_outputs.append(current)
# add 4 blocks with params [6, 12, 32, 32]
for i in range(1, 5):
block_outputs = dense_de_block(weights, current, i, block_outputs)
current = block_outputs[-1]
# add norm5 and conv1(which is 'conv2' in the original paper)
# 9 layer totally
layer_norm5 = features + 'norm5/'
with tf.variable_scope("norm5"):
n5_w = weights[layer_norm5 + sub_l[0]]
n5_b = weights[layer_norm5 + sub_l[1]]
n5_m = weights[layer_norm5 + sub_l[2]]
n5_v = weights[layer_norm5 + sub_l[3]]
conv1_kernels = weights['conv1/weight']
current = utils.batch_norm_de(current,
n5_m,
n5_v,
n5_b,
n5_w,
scope='bn')
current = utils.conv2d_de(current, conv1_kernels, 1, 1, name='conv1')
# batchnorm of norm5
layer_bn = 'bn/'
bn_w = weights[layer_bn + sub_l[0]]
bn_b = weights[layer_bn + sub_l[1]]
bn_m = weights[layer_bn + sub_l[2]]
bn_v = weights[layer_bn + sub_l[3]]
current = utils.batch_norm_de(current,
bn_m,
bn_v,
bn_b,
bn_w,
scope='bn')
block_outputs.append(current)
# add 4 upsampling blocks
for i in range(1, 5):
current = utils.up_block_de(block_outputs[-1], weights, i)
block_outputs.append(current)
# add conv2(which is 'conv3' in the original paper)
conv2_w = weights['conv2/weight']
conv2_w = conv2_w.transpose((2, 3, 1, 0))
conv2_b = weights['conv2/bias']
current = utils.conv2d_bias_de(current,
conv2_w,
kernel_size=3,
stride=1,
name='conv2',
padding=1,
bias=conv2_b)
block_outputs.append(current)
# the ouput of the base net is with idx 10
refine_block_inputs = []
refine_block_outputs = []
for i in range(1, 5):
refine_block_inputs.append(block_outputs[i])
refine_shape = block_outputs[-1].get_shape().as_list()
with tf.variable_scope("refine"):
for i in range(1, 5):
current = utils.up_block_refine_de(refine_block_inputs[i - 1],
refine_weights,
refine_shape[2],
refine_shape[1], i)
refine_block_outputs.append(current)
#conv1_input = refine_block_outputs[0]
#for i in range(1,4):
# conv1_input = tf.concat([conv1_input, refine_block_outputs[i]], 3)
conv1_input = tf.concat([
refine_block_outputs[0], refine_block_outputs[1],
refine_block_outputs[2], refine_block_outputs[3]
], 3)
conv1_w = refine_weights['conv1/weight']
rbn_m = refine_weights['bn1/running_mean']
rbn_v = refine_weights['bn1/running_var']
rbn_b = refine_weights['bn1/bias']
rbn_w = refine_weights['bn1/weight']
res = utils.conv2d_de(conv1_input,
conv1_w,
5,
1,
name='conv1',
padding="SAME")
res = utils.batch_norm_de(res,
rbn_m,
rbn_v,
rbn_b,
rbn_w,
scope='bn1')
res = utils.relu_de(res, name='relu1')
refine_block_outputs.append(res)
res = tf.concat([res, block_outputs[-1]], 3)
conv2_w = refine_weights['conv2/weight']
rbn_m = refine_weights['bn2/running_mean']
rbn_v = refine_weights['bn2/running_var']
rbn_b = refine_weights['bn2/bias']
rbn_w = refine_weights['bn2/weight']
res = utils.conv2d_de(res,
conv2_w,
5,
1,
name='conv2',
padding="SAME")
res = utils.batch_norm_de(res,
rbn_m,
rbn_v,
rbn_b,
rbn_w,
scope='bn2')
res = utils.relu_de(res, name='relu2')
refine_block_outputs.append(res)
conv3_w = refine_weights['conv3/weight']
rbn_m = refine_weights['bn3/running_mean']
rbn_v = refine_weights['bn3/running_var']
rbn_b = refine_weights['bn3/bias']
rbn_w = refine_weights['bn3/weight']
res = utils.conv2d_de(res,
conv3_w,
5,
1,
name='conv3',
padding="SAME")
res = utils.batch_norm_de(res,
rbn_m,
rbn_v,
rbn_b,
rbn_w,
scope='bn3')
res = utils.relu_de(res, name='relu3')
refine_block_outputs.append(res)
conv4_w = refine_weights['conv4/weight']
conv4_w = conv4_w.transpose((2, 3, 1, 0))
conv4_b = refine_weights['conv4/bias']
res = utils.conv2d_bias_de(res,
conv4_w,
kernel_size=5,
stride=1,
name='prediction',
padding=1,
bias=conv4_b)
refine_block_outputs.append(res)
## block_outputs.append(res)
## res = misc.imresize(res,(org_height, org_width))
#return block_outputs
return res
def inference(image, keep_prob):
"""
Semantic segmentation network definition
:param image: input image. Should have values in range 0-255
:param keep_prob:
:return:
"""
print("setting up vgg initialized conv layers ...")
model_data = utils.get_model_data(FLAGS.model_dir, MODEL_URL)
mean = model_data['normalization'][0][0][0]
mean_pixel = np.mean(mean, axis=(0, 1))
weights = np.squeeze(model_data['layers'])
#processed_image = utils.process_image(image, mean_pixel)
with tf.variable_scope("inference"):
image_net = vgg_net(weights, image)
conv_final_layer = image_net["conv5_3"]
pool5 = utils.max_pool_2x2(conv_final_layer)
W6 = utils.weight_variable([7, 7, 512, 4096], name="W6")
b6 = utils.bias_variable([4096], name="b6")
conv6 = utils.conv2d_basic(pool5, W6, b6)
relu6 = tf.nn.relu(conv6, name="relu6")
if FLAGS.debug:
utils.add_activation_summary(relu6)
relu_dropout6 = tf.nn.dropout(relu6, keep_prob=keep_prob)
W7 = utils.weight_variable([1, 1, 4096, 4096], name="W7")
b7 = utils.bias_variable([4096], name="b7")
conv7 = utils.conv2d_basic(relu_dropout6, W7, b7)
relu7 = tf.nn.relu(conv7, name="relu7")
if FLAGS.debug:
utils.add_activation_summary(relu7)
relu_dropout7 = tf.nn.dropout(relu7, keep_prob=keep_prob)
W8 = utils.weight_variable([1, 1, 4096, NUM_OF_CLASSESS], name="W8")
b8 = utils.bias_variable([NUM_OF_CLASSESS], name="b8")
conv8 = utils.conv2d_basic(relu_dropout7, W8, b8)
# annotation_pred1 = tf.argmax(conv8, dimension=3, name="prediction1")
# now to upscale to actual image size
deconv_shape1 = image_net["pool4"].get_shape()
W_t1 = utils.weight_variable([4, 4, deconv_shape1[3].value, NUM_OF_CLASSESS], name="W_t1")
b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1")
conv_t1 = utils.conv2d_transpose_strided(conv8, W_t1, b_t1, output_shape=tf.shape(image_net["pool4"]))
fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1")
deconv_shape2 = image_net["pool3"].get_shape()
W_t2 = utils.weight_variable([4, 4, deconv_shape2[3].value, deconv_shape1[3].value], name="W_t2")
b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2")
conv_t2 = utils.conv2d_transpose_strided(fuse_1, W_t2, b_t2, output_shape=tf.shape(image_net["pool3"]))
fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2")
shape = tf.shape(image)
deconv_shape3 = tf.stack([shape[0], shape[1], shape[2], NUM_OF_CLASSESS])
W_t3 = utils.weight_variable([16, 16, NUM_OF_CLASSESS, deconv_shape2[3].value], name="W_t3")
b_t3 = utils.bias_variable([NUM_OF_CLASSESS], name="b_t3")
conv_t3 = utils.conv2d_transpose_strided(fuse_2, W_t3, b_t3, output_shape=deconv_shape3, stride=8)
annotation_pred = tf.argmax(conv_t3, dimension=3, name="prediction")
return tf.expand_dims(annotation_pred, dim=3), conv_t3
def myinference_pretrained_weights(image, keep_prob, p="valid"):
# print("setting up vgg initialized conv layers ...")
model_data = utils.get_model_data(FLAGS.model_dir, MODEL_URL)
mean = model_data['normalization'][0][0][0]
weights = np.squeeze(model_data['layers'])
with tf.variable_scope("inference"):
image_net = vgg_net(weights, image)
# image_net = myvgg(image)
conv_final_layer = image_net["conv5_3"]
pool5 = tf.layers.max_pooling2d(conv_final_layer, 2, 2)
conv6 = tf.layers.conv2d(inputs=pool5,
filters=4096,
kernel_size=7,
padding=p,
activation=tf.nn.relu)
relu_dropout6 = tf.nn.dropout(conv6, keep_prob=keep_prob)
conv7 = tf.layers.conv2d(inputs=relu_dropout6,
filters=4096,
kernel_size=1,
padding=p,
activation=tf.nn.relu)
if FLAGS.debug:
utils.add_activation_summary(conv7)
relu_dropout7 = tf.nn.dropout(conv7, keep_prob=keep_prob)
#### first deconv
score = tf.layers.conv2d(inputs=relu_dropout7,
filters=2,
padding=p,
kernel_size=1)
# score2
conv_t1 = tf.layers.conv2d_transpose(inputs=score,
filters=2,
padding=p,
kernel_size=4,
strides=2)
score_pool4 = tf.layers.conv2d(inputs=image_net["pool4"],
filters=2,
kernel_size=1,
padding=p)
score_fused = utils.crop_and_add(score_pool4, conv_t1)
#### second deconv
# score4
conv_t2 = tf.layers.conv2d_transpose(inputs=score_fused,
filters=2,
padding=p,
kernel_size=4,
strides=2,
use_bias=False)
score_pool3 = tf.layers.conv2d(inputs=image_net["pool3"],
filters=2,
kernel_size=1,
padding=p)
score_fused2 = utils.crop_and_add(score_pool3, conv_t2)
# ### final deconv
# # upsample
conv_t3 = tf.layers.conv2d_transpose(inputs=score_fused2,
filters=2,
padding=p,
kernel_size=16,
strides=8,
use_bias=False)
mask = utils.crop_and_add(conv_t3, image, to_add=False)
# this is not needed
annotation_pred = tf.argmax(mask, dimension=3, name="prediction")
return annotation_pred, mask