def base_CNN(self, state):
with tf.variable_scope(self.scope):
num_channels = get_num_channels(state)
with tf.variable_scope('conv_1'):
x = convolution(state, [4,4,num_channels,8], strides=2, padding="VALID")
x = tf.layers.batch_normalization(x, momentum=0.99, epsilon=0.001,center=True, scale=True,training=self.train_phase)
x = tf.nn.relu(x)
with tf.variable_scope('conv_2'):
x = convolution(x, [3,3,8,16], strides=2, padding="VALID")
x = tf.layers.batch_normalization(x, momentum=0.99, epsilon=0.001,center=True, scale=True,training=self.train_phase)
x = tf.nn.relu(x)
with tf.variable_scope('conv_3'):
x = convolution(x, [3,3,16,32], strides=2, padding="VALID")
x = tf.layers.batch_normalization(x, momentum=0.99, epsilon=0.001,center=True, scale=True,training=self.train_phase)
x = tf.nn.relu(x)
with tf.variable_scope('conv_4'):
x = convolution(x, [3,3,32,64], strides=2, padding="VALID")
x = tf.layers.batch_normalization(x, momentum=0.99, epsilon=0.001,center=True, scale=True,training=self.train_phase)
x = tf.nn.relu(x)
with tf.variable_scope('conv_5'):
x = convolution(x, [3,3,64,128], strides=1, padding="VALID")
x = tf.layers.batch_normalization(x, momentum=0.99, epsilon=0.001,center=True, scale=True,training=self.train_phase)
x = tf.nn.relu(x)
#with tf.variable_scope('conv_6'):
# x = convolution(x, [3,3,128,256], strides=1, padding="VALID")
# x = tf.layers.batch_normalization(x, momentum=0.99, epsilon=0.001,center=True, scale=True,training=self.train_phase)
# x = tf.nn.relu(x)
x = tf.reduce_mean(x, axis=[1,2])
x = tf.keras.layers.Dense(units=128)(x)
#x = tf.nn.dropout(x, self.keep_prob)
logits = tf.keras.layers.Dense(units=6)(x)
return logits
def discriminator(self, input, training, reuse):
"""
The discriminator is responsible for taking an image and predicting whether or not it contains a generated patch
Both the narrow and wide path take in the same input image, and merge together before carrying out a couple
more convolutions and making a final decision
:argument
input: The input could either be a real image or one containing a patch that was generated
training: Whether the network is training or not
reuse: Whether the weights in all the layers are to be reused
:returns
output: Prediction on whether or not the discriminator believes an image contains a generated patch or
not. If output is 0, then the network is 100% certain the image is real. If it is 1, then it is
100% certain it contains a patch
"""
with tf.variable_scope("discriminator", reuse=reuse):
# Narrow Path_______________________________________________________________________________________________
layer = convolution(input,
128, (5, 5), (1, 1),
1,
training,
reuse,
dilation_rate=(3, 3))
layer = convolution(layer, 128, (5, 5), (2, 2), 2, training, reuse)
layer = convolution(layer,
128, (5, 5), (1, 1),
3,
training,
reuse,
dilation_rate=(3, 3))
layer = convolution(layer,
128, (5, 5), (1, 1),
4,
training,
reuse,
dilation_rate=(4, 4))
layer = convolution(layer, 128, (5, 5), (2, 2), 5, training, reuse)
layer = convolution(layer, 128, (5, 5), (2, 2), 6, training, reuse)
layer = convolution(layer,
128, (5, 5), (1, 1),
7,
training,
reuse,
dilation_rate=(4, 4),
batch_norm=True)
# Wide Path_________________________________________________________________________________________________
layer_wide = convolution(input, 128, (10, 10), (1, 1), 8, training,
reuse)
layer_wide = convolution(layer_wide, 128, (10, 10), (1, 1), 9,
training, reuse)
layer_wide = convolution(layer_wide, 128, (9, 9), (2, 2), 10,
training, reuse)
layer_wide = convolution(layer_wide,
128, (9, 9), (1, 1),
11,
training,
reuse,
padding='VALID')
layer_wide = convolution(layer_wide, 128, (5, 5), (1, 1), 12,
training, reuse)
layer_wide = convolution(layer_wide,
128, (5, 5), (1, 1),
13,
training,
reuse,
padding='VALID',
batch_norm=True)
#___________________________________________________________________________________________________________
layer = tf.concat([layer, layer_wide], 3)
layer = convolution(layer, 128, (5, 5), (1, 1), 14, training,
reuse)
layer = convolution(layer,
128, (5, 5), (1, 1),
15,
training,
reuse,
batch_norm=True)
layer = tf.reshape(layer, (-1, 4 * 4 * 128))
output = tf.layers.dense(layer, 1, name='d_16')
return output
def get_model(point_cloud, is_training):
""" Classification TFN, input is BxNx3, output Bx10 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
# radial basis functions
rbf_low = 0.0
rbf_high = 3.5
rbf_count = 4
rbf_spacing = (rbf_high - rbf_low) / rbf_count
centers = tf.cast(tf.lin_space(rbf_low, rbf_high, rbf_count), FLOAT_TYPE)
# rij : [None, N, N, 3]
rij = utils.difference_matrix(point_cloud)
# dij : [None, N, N]
dij = utils.distance_matrix(point_cloud)
# rbf : [None, N, N, rbf_count]
gamma = 1. / rbf_spacing
rbf = tf.exp(-gamma * tf.square(tf.expand_dims(dij, axis=-1) - centers))
# channels
layer_dims = [1, 8, 8, 8]
num_layers = len(layer_dims) - 1
# embed : [None, N, layer1_dim, 1]
with tf.variable_scope(None, "embed"):
embed = layers.self_interaction_layer_without_biases(
tf.ones(shape=(tf.shape(point_cloud)[0], num_point, 1, 1)),
layer_dims[0])
input_tensor_list = {0: [embed]}
for layer, layer_dim in enumerate(layer_dims[1:]):
with tf.variable_scope(None,
"layer" + str(layer),
values=[input_tensor_list]):
input_tensor_list = layers.convolution(input_tensor_list, rbf, rij)
input_tensor_list = layers.concatenation(input_tensor_list)
input_tensor_list = layers.self_interaction(
input_tensor_list, layer_dim)
input_tensor_list = layers.nonlinearity(input_tensor_list)
tfn_scalars = input_tensor_list[0][0]
tfn_output_shape = tfn_scalars.get_shape().as_list()
axis_to_squeeze = [i for i, e in enumerate(tfn_output_shape) if e == 1]
tfn_output = tf.reduce_mean(tf.squeeze(tfn_scalars, axis=axis_to_squeeze),
axis=1)
fully_connected_layer = tf.get_variable(
"fully_connected_weights", [tfn_output_shape[-2], NUM_CLASSES],
dtype=FLOAT_TYPE)
output_biases = tf.get_variable("output_biases", [NUM_CLASSES],
dtype=FLOAT_TYPE)
# output : [None, NUM_CLASSES]
output = tf.einsum("xy,hx->hy", fully_connected_layer,
tfn_output) + output_biases
return output, end_points
def generator(self,
input,
surrounding_region,
sparse,
training,
reuse=False):
"""
The generator is responsible for taking a masked image and producing a plausible patch. Both the narrow and wide
paths take the same image as their input, while the shallow layer takes segments from the top, bottom, left and
right regions which surround the masked selection.
:argument
input: Image input containing a masked out region
surrounding_region: Pixels surrounding the masked out region, cropped to the input size of the
discriminator
sparse: A sparse tensor used to extract the generated patch from the generator output and
merge it into the surrounding pixels of the original image
training: Whether the network is currently training or not
reuse: Whether the learnt weights should be reused or not
:returns
image: Contains the patch along with the surrounding pixels in the original image cropped
to the input size of the discriminator
patch: The generated patch
"""
with tf.variable_scope("generator", reuse=reuse):
# Shallow Path______________________________________________________________________________________________
#
# The shallow path consists of two convolution layers.
# The first layer takes segments from above and below the image which are then concatenated along the Y-axis
# The second take segments from the left and right of the image which are concatenated along the X-axis
# Following the process of a single convolution, these layers are concatenated with the output from the
# narrow and wide path a couple of layers prior to the output layer
y_axis = 1
x_axis = 2
image_top = tf.slice(input, [0, 0, 8, 0], [-1, 16, 32, channels])
image_bottom = tf.slice(input, [0, 32, 8, 0],
[-1, 16, 32, channels])
layer_shallow = tf.concat([image_top, image_bottom], y_axis)
layer_shallow = convolution(layer_shallow,
128, (3, 3), (1, 1),
2,
training,
reuse,
dilation_rate=(3, 3),
batch_norm=True)
image_left = tf.slice(input, [0, 8, 0, 0], [-1, 32, 16, channels])
image_right = tf.slice(input, [0, 8, 32, 0],
[-1, 32, 16, channels])
layer_shallow_ = tf.concat([image_left, image_right], x_axis)
layer_shallow_ = convolution(layer_shallow_,
128, (3, 3), (1, 1),
1,
training,
reuse,
dilation_rate=(3, 3),
batch_norm=True)
# Narrow Path_______________________________________________________________________________________________
layer = convolution(input,
128, (5, 5), (1, 1),
3,
training,
reuse,
dilation_rate=(3, 3))
layer = convolution(layer,
128, (5, 5), (1, 1),
4,
training,
reuse,
dilation_rate=(3, 3))
layer = convolution(layer, 128, (3, 3), (2, 2), 5, training, reuse)
layer = convolution(layer,
128, (3, 3), (1, 1),
6,
training,
reuse,
dilation_rate=(3, 3))
layer = conv_transpose(layer,
128, (5, 5), (1, 1),
7,
training,
padding="VALID")
layer = convolution(layer,
128, (3, 3), (1, 1),
8,
training,
reuse,
dilation_rate=(2, 2))
layer = convolution(layer,
128, (3, 3), (1, 1),
9,
training,
reuse,
dilation_rate=(4, 4))
layer = convolution(layer,
128, (3, 3), (1, 1),
10,
training,
reuse,
dilation_rate=(8, 8))
layer = conv_transpose(layer,
128, (5, 5), (1, 1),
11,
training,
padding="VALID")
layer = convolution(layer,
128, (5, 5), (1, 1),
12,
training,
reuse,
dilation_rate=(4, 4),
batch_norm=True)
# Wide Path_________________________________________________________________________________________________
layer_wide = convolution(input, 128, (8, 8), (1, 1), 13, training,
reuse)
layer_wide = convolution(layer_wide, 128, (8, 8), (1, 1), 14,
training, reuse)
layer_wide = convolution(layer_wide,
128, (9, 9), (1, 1),
15,
training,
reuse,
padding='VALID')
layer_wide = convolution(layer_wide,
128, (8, 8), (1, 1),
16,
training,
reuse,
dilation_rate=(6, 6))
layer_wide = convolution(layer_wide,
128, (9, 9), (1, 1),
17,
training,
reuse,
padding='VALID')
layer_wide = convolution(layer_wide,
128, (8, 8), (1, 1),
18,
training,
reuse,
dilation_rate=(6, 6))
layer_wide = convolution(layer_wide,
128, (8, 8), (1, 1),
19,
training,
reuse,
batch_norm=True)
#___________________________________________________________________________________________________________
layer = tf.concat(
[layer, layer_wide, layer_shallow, layer_shallow_], 3)
layer = convolution(layer, 128, (3, 3), (1, 1), 20, training,
reuse)
layer = convolution(layer,
64, (3, 3), (1, 1),
21,
training,
reuse,
batch_norm=True)
layer = convolution(layer,
channels, (5, 5), (1, 1),
22,
training,
reuse,
activation=False)
layer = tf.nn.sigmoid(layer)
patch = layer * 255
image = self.merge_original_image_with_generated(
surrounding_region, patch, sparse)
patch = tf.slice(image,
[0, D_patch_margin_size, D_patch_margin_size, 0],
[-1, patch_width, patch_width, channels])
return image, patch