def _discriminator(self,
input_images,
dims,
train_phase,
activation=tf.nn.relu,
scope_name="discriminator",
scope_reuse=False):
N = len(dims)
with tf.variable_scope(scope_name) as scope:
if scope_reuse:
scope.reuse_variables()
h = input_images
skip_bn = True # First layer of discriminator skips batch norm
for index in range(N - 2):
W = utils.weight_variable([4, 4, dims[index], dims[index + 1]],
name="W_%d" % index)
b = tf.zeros([dims[index + 1]])
h_conv = utils.conv2d_strided(h, W, b)
if skip_bn:
h_bn = h_conv
skip_bn = False
else:
#d_bn = ops.batch_norm(name='d_bn{0}'.format(index))
h_bn = d_bn(h_conv, train=train_phase)
h = activation(h_bn, name="h_%d" % index)
utils.add_activation_summary(h)
W_pred = utils.weight_variable([4, 4, dims[-2], dims[-1]],
name="W_pred")
b = tf.zeros([dims[-1]])
h_pred = utils.conv2d_strided(h, W_pred, b)
return None, h_pred, None # Return the last convolution output. None values are returned to maintatin disc from other GAN
def _discriminator(self, input_images, dims, train_phase, activation=tf.nn.relu, scope_name="discriminator",
scope_reuse=False):
N = len(dims)
with tf.variable_scope(scope_name) as scope:
if scope_reuse:
scope.reuse_variables()
h = input_images
skip_bn = True # First layer of discriminator skips batch norm
for index in range(N - 2):
W = utils.weight_variable([5, 5, dims[index], dims[index + 1]], name="W_%d" % index)
b = utils.bias_variable([dims[index + 1]], name="b_%d" % index)
h_conv = utils.conv2d_strided(h, W, b)
if skip_bn:
h_bn = h_conv
skip_bn = False
else:
h_bn = utils.batch_norm(h_conv, dims[index + 1], train_phase, scope="disc_bn%d" % index)
h = activation(h_bn, name="h_%d" % index)
utils.add_activation_summary(h)
shape = h.get_shape().as_list()
image_size = self.resized_image_size // (2 ** (N - 2)) # dims has input dim and output dim
h_reshaped = tf.reshape(h, [self.batch_size, image_size * image_size * shape[3]])
W_pred = utils.weight_variable([image_size * image_size * shape[3], dims[-1]], name="W_pred")
b_pred = utils.bias_variable([dims[-1]], name="b_pred")
h_pred = tf.matmul(h_reshaped, W_pred) + b_pred
return tf.nn.sigmoid(h_pred), h_pred, h
def vgg_net(weights, image):
layers = ('conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1', 'conv2_1',
'relu2_1', 'conv2_2', 'relu2_2', 'pool2', 'conv3_1', 'relu3_1',
'conv3_2', 'relu3_2', 'conv3_3', 'relu3_3', 'conv3_4', 'relu3_4',
'pool3', 'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3',
'relu4_3', 'conv4_4', 'relu4_4', 'pool4', 'conv5_1', 'relu5_1',
'conv5_2', 'relu5_2', 'conv5_3', 'relu5_3', 'conv5_4', 'relu5_4')
net = {}
current = image
for i, name in enumerate(layers):
kind = name[:4]
if kind == 'conv':
kernels, bias = weights[i][0][0][0][0]
# matconvnet: weights are [width, height, in_channels, out_channels]
# tensorflow: weights are [height, width, in_channels, out_channels]
kernels = utils.get_variable(np.transpose(kernels, (1, 0, 2, 3)),
name=name + "_w")
bias = utils.get_variable(bias.reshape(-1), name=name + "_b")
current = utils.conv2d_basic(current, kernels, bias)
elif kind == 'relu':
current = tf.nn.relu(current, name=name)
if FLAGS.debug:
utils.add_activation_summary(current)
elif kind == 'pool':
current = utils.avg_pool_2x2(current)
net[name] = current
return net
def _discriminator(self,
input_images,
dims,
train_phase,
activation=tf.nn.relu,
scope_name="discriminator",
scope_reuse=False):
N = len(dims)
with tf.variable_scope(scope_name) as scope:
if scope_reuse:
scope.reuse_variables()
h = input_images
skip_bn = True # First layer of discriminator skips batch norm
for index in range(N - 2):
W = utils.weight_variable([4, 4, dims[index], dims[index + 1]],
name="W_%d" % index)
b = tf.zeros([dims[index + 1]])
h_conv = utils.conv2d_strided(h, W, b)
if skip_bn:
h_bn = h_conv
skip_bn = False
else:
# h_bn = tf.contrib.layers.batch_norm(inputs=h_conv, decay=0.9, epsilon=1e-5, is_training=train_phase,
# scope="disc_bn%d" % index)
h_bn = utils.batch_norm("disc_bn%d" % index,
h_conv,
True,
'NHWC',
train_phase,
bn_epsilon=1e-5,
bn_ema=0.9)
# h_bn = utils.batch_norm(h_conv, dims[index + 1], train_phase, scope="disc_bn%d" % index)
h = activation(h_bn, name="h_%d" % index)
utils.add_activation_summary(h)
W_pred = utils.weight_variable([4, 4, dims[-2], dims[-1]],
name="W_pred")
b = tf.zeros([dims[-1]])
h_pred = utils.conv2d_strided(h, W_pred, b)
return None, h_pred, None # Return the last convolution output. None values are returned to maintatin disc from other GAN
def _generator(self, z, dims, train_phase, activation=tf.nn.relu, scope_name="generator"):
N = len(dims)
image_size = self.resized_image_size // (2 ** (N - 1))
with tf.variable_scope(scope_name) as scope:
W_z = utils.weight_variable([self.z_dim, dims[0] * image_size * image_size], name="W_z")
b_z = utils.bias_variable([dims[0] * image_size * image_size], name="b_z")
h_z = tf.matmul(z, W_z) + b_z
h_z = tf.reshape(h_z, [-1, image_size, image_size, dims[0]])
h_bnz = utils.batch_norm(h_z, dims[0], train_phase, scope="gen_bnz")
h = activation(h_bnz, name='h_z')
utils.add_activation_summary(h)
for index in range(N - 2):
image_size *= 2
W = utils.weight_variable([5, 5, dims[index + 1], dims[index]], name="W_%d" % index)
b = utils.bias_variable([dims[index + 1]], name="b_%d" % index)
deconv_shape = tf.stack([tf.shape(h)[0], image_size, image_size, dims[index + 1]])
h_conv_t = utils.conv2d_transpose_strided(h, W, b, output_shape=deconv_shape)
h_bn = utils.batch_norm(h_conv_t, dims[index + 1], train_phase, scope="gen_bn%d" % index)
h = activation(h_bn, name='h_%d' % index)
utils.add_activation_summary(h)
image_size *= 2
W_pred = utils.weight_variable([5, 5, dims[-1], dims[-2]], name="W_pred")
b_pred = utils.bias_variable([dims[-1]], name="b_pred")
deconv_shape = tf.stack([tf.shape(h)[0], image_size, image_size, dims[-1]])
h_conv_t = utils.conv2d_transpose_strided(h, W_pred, b_pred, output_shape=deconv_shape)
pred_image = tf.nn.tanh(h_conv_t, name='pred_image')
utils.add_activation_summary(pred_image)
return pred_image
def _generator(self,
z,
dims,
train_phase,
activation=tf.nn.relu,
scope_name="generator"):
N = len(dims)
image_size = self.resized_image_size // (2**(N - 1))
with tf.variable_scope(scope_name) as scope:
W_z = utils.weight_variable(
[self.z_dim, dims[0] * image_size * image_size], name="W_z")
h_z = tf.matmul(z, W_z)
h_z = tf.reshape(h_z, [-1, image_size, image_size, dims[0]])
# h_bnz = tf.contrib.layers.batch_norm(inputs=h_z, decay=0.9, epsilon=1e-5, is_training=train_phase,
# scope="gen_bnz")
# h_bnz = utils.batch_norm(h_z, dims[0], train_phase, scope="gen_bnz")
h_bnz = utils.batch_norm('gen_bnz', h_z, True, 'NHWC', train_phase)
h = activation(h_bnz, name='h_z')
utils.add_activation_summary(h)
for index in range(N - 2):
image_size *= 2
W = utils.weight_variable([4, 4, dims[index + 1], dims[index]],
name="W_%d" % index)
b = tf.zeros([dims[index + 1]])
deconv_shape = tf.stack(
[tf.shape(h)[0], image_size, image_size, dims[index + 1]])
h_conv_t = utils.conv2d_transpose_strided(
h, W, b, output_shape=deconv_shape)
# h_bn = tf.contrib.layers.batch_norm(inputs=h_conv_t, decay=0.9, epsilon=1e-5, is_training=train_phase,
# scope="gen_bn%d" % index)
# h_bn = utils.batch_norm(h_conv_t, dims[index + 1], train_phase, scope="gen_bn%d" % index)
h_bn = utils.batch_norm("gen_bn%d" % index, h_conv_t, True,
'NHWC', train_phase)
h = activation(h_bn, name='h_%d' % index)
utils.add_activation_summary(h)
image_size *= 2
W_pred = utils.weight_variable([4, 4, dims[-1], dims[-2]],
name="W_pred")
b = tf.zeros([dims[-1]])
deconv_shape = tf.stack(
[tf.shape(h)[0], image_size, image_size, dims[-1]])
h_conv_t = utils.conv2d_transpose_strided(
h, W_pred, b, output_shape=deconv_shape)
pred_image = tf.nn.tanh(h_conv_t, name='pred_image')
utils.add_activation_summary(pred_image)
return pred_image
def segmentation(image, keep_prob):
"""
图像语义分割模型定义
Parameters
----------
image: 输入图像，每个通道的像素值为0到255
keep_prob: 防止过拟合的dropout参数
Returns
-------
"""
print("setting up vgg initialized conv layers ...")
model_data = utils.get_model_data(FLAGS.model_dir)
# vgg模型的权重值
weights = np.squeeze(model_data['layers'])
# 计算图片像素值的均值, 然后对图像加上均值
mean = model_data['normalization'][0][0][0]
mean_pixel = np.mean(mean, axis=(0, 1))
processed_image = utils.process_image(image, mean_pixel)
# 共享变量名空间-segmentation
with tf.variable_scope("segmentation"):
image_net = vgg_net(weights, processed_image)
conv_final_layer = image_net["conv5_3"]
pool5 = utils.max_pool_2x2(conv_final_layer)
# 全连接层用卷积层来代替
W6 = utils.weight_variable([7, 7, 512, 4096], name = "W6")
b6 = utils.bias_variable([4096], name="b6")
conv6 = utils.conv2d_basic(pool5, W6, b6)
relu6 = tf.nn.relu(conv6, name="relu6")
if FLAGS.debug:
utils.add_activation_summary(relu6)
# 随机去掉一些神经元防止过拟合
relu_dropout6 = tf.nn.dropout(relu6, keep_prob=keep_prob)
W7 = utils.weight_variable([1, 1, 4096, 4096], name="W7")
b7 = utils.bias_variable([4096], name="b7")
conv7 = utils.conv2d_basic(relu_dropout6, W7, b7)
relu7 = tf.nn.relu(conv7, name="relu7")
if FLAGS.debug:
utils.add_activation_summary(relu7)
relu_dropout7 = tf.nn.dropout(relu7, keep_prob=keep_prob)
W8 = utils.weight_variable([1, 1, 4096, NUM_OF_CLASSESS], name="W8")
b8 = utils.bias_variable([NUM_OF_CLASSESS], name="b8")
conv8 = utils.conv2d_basic(relu_dropout7, W8, b8)
# annotation_pred1 = tf.argmax(conv8, dimension=3, name="prediction1")
# now to upscale to actual image size
deconv_shape1 = image_net["pool4"].get_shape()
W_t1 = utils.weight_variable([4, 4, deconv_shape1[3].value, NUM_OF_CLASSESS], name="W_t1")
b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1")
conv_t1 = utils.conv2d_transpose_strided(conv8, W_t1, b_t1, output_shape=tf.shape(image_net["pool4"]))
fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1")
deconv_shape2 = image_net["pool3"].get_shape()
W_t2 = utils.weight_variable([4, 4, deconv_shape2[3].value, deconv_shape1[3].value], name="W_t2")
b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2")
conv_t2 = utils.conv2d_transpose_strided(fuse_1, W_t2, b_t2, output_shape=tf.shape(image_net["pool3"]))
fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2")
shape = tf.shape(image)
deconv_shape3 = tf.stack([shape[0], shape[1], shape[2], NUM_OF_CLASSESS])
W_t3 = utils.weight_variable([16, 16, NUM_OF_CLASSESS, deconv_shape2[3].value], name="W_t3")
b_t3 = utils.bias_variable([NUM_OF_CLASSESS], name = "b_t3")
conv_t3 = utils.conv2d_transpose_strided(fuse_2, W_t3, b_t3, output_shape = deconv_shape3, stride = 8)
# 预测结果层
annotation_pred = tf.argmax(conv_t3, dimension = 3, name = "prediction")
return tf.expand_dims(annotation_pred, dim = 3), conv_t3
def inference(image, keep_prob, train=False):
"""
Semantic segmentation network definition
:param image: input image. Should have values in range 0-255
:param keep_prob:
:return:
"""
print("setting up vgg initialized conv layers ...")
model_data = utils.get_model_data(FLAGS.model_dir, MODEL_URL)
mean = model_data['normalization'][0][0][0]
mean_pixel = np.mean(mean, axis=(0, 1))
weights = np.squeeze(model_data['layers'])
# accounts for the mean being subtracted from the image
processed_image = utils.process_image(image, mean_pixel)
with tf.variable_scope("inference"):
image_net = vgg_net(weights, processed_image)
conv_final_layer = image_net["conv5_3"]
pool5 = utils.max_pool_2x2(conv_final_layer)
W6 = utils.weight_variable([7, 7, 512, 4096], name="W6")
b6 = utils.bias_variable([4096], name="b6")
conv6 = utils.conv2d_basic(pool5, W6, b6)
relu6 = tf.nn.relu(conv6, name="relu6")
if FLAGS.debug:
utils.add_activation_summary(relu6)
if train:
relu6 = tf.nn.dropout(relu6, keep_prob=keep_prob)
W7 = utils.weight_variable([1, 1, 4096, 4096], name="W7")
b7 = utils.bias_variable([4096], name="b7")
conv7 = utils.conv2d_basic(relu6, W7, b7)
relu7 = tf.nn.relu(conv7, name="relu7")
if FLAGS.debug:
utils.add_activation_summary(relu7)
if train:
relu7 = tf.nn.dropout(relu7, keep_prob=keep_prob)
W8 = utils.weight_variable([1, 1, 4096, FLAGS.NUM_OF_CLASSES], name="W8")
b8 = utils.bias_variable([FLAGS.NUM_OF_CLASSES], name="b8")
conv8 = utils.conv2d_basic(relu7, W8, b8)
# annotation_pred1 = tf.argmax(conv8, dimension=3, name="prediction1")
# now to upscale to actual image size
deconv_shape1 = image_net["pool4"].get_shape()
W_t1 = utils.weight_variable([4, 4, deconv_shape1[3].value, FLAGS.NUM_OF_CLASSES], name="W_t1")
b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1")
conv_t1 = utils.conv2d_transpose_strided(conv8, W_t1, b_t1, output_shape=tf.shape(image_net["pool4"]))
fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1")
deconv_shape2 = image_net["pool3"].get_shape()
W_t2 = utils.weight_variable([4, 4, deconv_shape2[3].value, deconv_shape1[3].value], name="W_t2")
b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2")
conv_t2 = utils.conv2d_transpose_strided(fuse_1, W_t2, b_t2, output_shape=tf.shape(image_net["pool3"]))
fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2")
shape = tf.shape(image)
deconv_shape3 = tf.pack([shape[0], shape[1], shape[2], FLAGS.NUM_OF_CLASSES])
W_t3 = utils.weight_variable([16, 16, FLAGS.NUM_OF_CLASSES, deconv_shape2[3].value], name="W_t3")
b_t3 = utils.bias_variable([FLAGS.NUM_OF_CLASSES], name="b_t3")
conv_t3 = utils.conv2d_transpose_strided(fuse_2, W_t3, b_t3, output_shape=deconv_shape3, stride=8)
annotation_pred = tf.argmax(conv_t3, dimension=3, name="prediction")
return tf.expand_dims(annotation_pred, dim=3), conv_t3
def _generator(self,
z,
dims,
train_phase,
activation=tf.nn.relu,
scope_name="generator"):
N = len(dims)
image_size = self.resized_image_size // (2**(N - 1))
input_labels = tf.cond(
train_phase, lambda: self.labels, lambda: tf.one_hot(
self.class_num * tf.ones(shape=self.batch_size, dtype=tf.int32
), self.num_cls))
with tf.variable_scope(scope_name) as scope:
W_ebd = utils.weight_variable([self.num_cls, self.z_dim],
name='W_ebd')
b_ebd = utils.bias_variable([self.z_dim], name='b_ebd')
h_ebd = tf.matmul(input_labels, W_ebd) + b_ebd
h_bnebd = utils.batch_norm(h_ebd,
self.z_dim,
train_phase,
scope='gen_bnebd')
h_ebd = activation(h_bnebd, name='h_bnebd')
#h_ebd = activation(h_ebd, name='h_ebd')
utils.add_activation_summary(h_ebd)
h_zebd = tf.multiply(h_ebd, z) #for TensorFlow 1.0
#h_zebd = tf.mul(h_ebd, z)
W_z = utils.weight_variable(
[self.z_dim, dims[0] * image_size * image_size], name="W_z")
b_z = utils.bias_variable([dims[0] * image_size * image_size],
name="b_z")
h_z = tf.matmul(h_zebd, W_z) + b_z
h_z = tf.reshape(h_z, [-1, image_size, image_size, dims[0]])
h_bnz = utils.batch_norm(h_z,
dims[0],
train_phase,
scope="gen_bnz")
h = activation(h_bnz, name='h_z')
utils.add_activation_summary(h)
for index in range(N - 2):
image_size *= 2
W = utils.weight_variable([4, 4, dims[index + 1], dims[index]],
name="W_%d" % index)
b = utils.bias_variable([dims[index + 1]], name="b_%d" % index)
deconv_shape = tf.stack(
[tf.shape(h)[0], image_size, image_size, dims[index + 1]])
h_conv_t = utils.conv2d_transpose_strided(
h, W, b, output_shape=deconv_shape)
h_bn = utils.batch_norm(h_conv_t,
dims[index + 1],
train_phase,
scope="gen_bn%d" % index)
h = activation(h_bn, name='h_%d' % index)
utils.add_activation_summary(h)
image_size *= 2
W_pred = utils.weight_variable([4, 4, dims[-1], dims[-2]],
name="W_pred")
b_pred = utils.bias_variable([dims[-1]], name="b_pred")
deconv_shape = tf.stack(
[tf.shape(h)[0], image_size, image_size, dims[-1]])
h_conv_t = utils.conv2d_transpose_strided(
h, W_pred, b_pred, output_shape=deconv_shape)
pred_image = tf.nn.tanh(h_conv_t, name='pred_image')
utils.add_activation_summary(pred_image)
return pred_image #, input_labels
