def trans_net(point_cloud,
k=20,
is_training=False,
bn_decay=None,
scope='transform_net1'):
# input (B, N, 3) return(B, N, 3)
adj_matrix = tf_util.pairwise_distance(point_cloud)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k)
with tf.variable_scope(scope) as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
return point_cloud_transformed
def get_model_other(point_cloud, is_training, bn_decay=None):
"""
B: batch size;
N: number of points,
C: channels;
k: number of nearest neighbors
point_cloud: B*N*C
"""
end_points = {}
minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1))
batch_size = point_cloud.get_shape()[0] # .value
# # 1. graph for first EdgeConv B N C=6
adj_matrix = tf_util.pairwise_distance(
point_cloud[:, :, :para.dim]) # B N C=6 => B*N*N
# adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:para.dim]) # B N C=6 => B*N*N
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn1'] = allSF_dist
point_cloud = tf.expand_dims(point_cloud[:, :, :para.dim], axis=-2)
# point_cloud = tf.expand_dims(point_cloud[:, :, 1:para.dim], axis=-2)
edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn1',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net1 = net
# # 2. graph for second EdgeConv B N C=64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn2'] = allSF_dist
# net: B*N*1*6+64=71
net = tf.concat([point_cloud, net1], axis=-1)
# edge_feature: B*N*k*142
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn2',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net2 = net
# 3. graph for third EdgeConv B N C=64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn3'] = allSF_dist
# net: B*N*1*6+64+64=134
net = tf.concat([point_cloud, net1, net2], axis=-1)
# edge_feature: B*N*k*268
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn3',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net3 = net
# 4. graph for fourth EdgeConv B N C=64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn4'] = allSF_dist
# net: B*N*1*6+64+64+64=198
net = tf.concat([point_cloud, net1, net2, net3], axis=-1)
# edge_feature: B*N*k*396
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn4',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net4 = net
# input: B*N*1*6+64+64+128+128 = 326 => net: B*N*1*1024
net = tf_util.conv2d(tf.concat([point_cloud, net1, net2, net3, net4],
axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='agg',
bn_decay=bn_decay)
# net: B*1*1*1024
# SF_features = tf.gather(net, indices=minSF, axis=1, batch_dims=1)
net = tf.reduce_max(net, axis=1, keepdims=True)
# SF_all = tf.concat([SF_features,net], axis=-1)
# net: B*1024
net = tf.squeeze(net)
# net: B*2048
# net = tf.squeeze(SF_all)
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
print(net.get_shape())
end_points['global_feature'] = net
# Fully connected end_points: classifier
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['fc1'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
end_points['fc2'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net,
para.outputClassN,
activation_fn=None,
scope='fc3')
end_points['fc3'] = net
return net, end_points
def create_encoder(self, point_cloud, npts):
point_cloud = tf.reshape(point_cloud, (BATCH_SIZE, NUM_POINT, 3))
adj_matrix = tf_util.pairwise_distance(point_cloud)
nn_idx = tf_util.knn(adj_matrix, k=self.knn)
edge_feature = tf_util.get_edge_feature(point_cloud,
nn_idx=nn_idx,
k=self.knn)
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net_dgcnn(edge_feature,
self.is_training,
self.bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
adj_matrix = tf_util.pairwise_distance(point_cloud_transformed)
nn_idx = tf_util.knn(adj_matrix, k=self.knn)
edge_feature = tf_util.get_edge_feature(point_cloud_transformed,
nn_idx=nn_idx,
k=self.knn)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=self.is_training,
scope='dgcnn1',
bn_decay=self.bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net1 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=self.knn)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=self.knn)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=self.is_training,
scope='dgcnn2',
bn_decay=self.bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net2 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=self.knn)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=self.knn)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=self.is_training,
scope='dgcnn3',
bn_decay=self.bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net3 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=self.knn)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=self.knn)
net = tf_util.conv2d(edge_feature,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=self.is_training,
scope='dgcnn4',
bn_decay=self.bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net4 = net
net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=self.is_training,
scope='agg',
bn_decay=self.bn_decay)
net = tf.reduce_max(net, axis=1, keep_dims=True)
features = tf.reshape(net, [BATCH_SIZE, -1])
return features
def get_graph_feature(x, k):
"""Torch: get_graph_feature = TF: adj_matrix + nn_idx + edge_feature"""
adj_matrix = tf_util.pairwise_distance(x)
nn_idx = tf_util.knn(adj_matrix, k=k)
x = tf_util.get_edge_feature(x, nn_idx=nn_idx, k=k)
return x
def get_model_other(point_cloud, is_training, bn_decay=None):
""" Classification DGCNN, input is BxNxC, output BxCls
B batch size
N number of points per pointcloud
C input channel: eg. x,y,z,SF,distance, minSF...
Cls output class number
"""
batch_size = point_cloud.get_shape()[0] # .value
end_points = {}
# get MinSF index
minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1))
# # 1. graph for transform net with only x,y,z
adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:4]) # B N C=3 => B N N
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(point_cloud[:, :, 1:4], nn_idx=nn_idx, k=para.k)
with tf.compat.v1.variable_scope('transform_net1') as sc:
transform = input_transform_net_dgcnn(edge_feature, is_training, bn_decay, K=3)
point_cloud_transform = tf.matmul(point_cloud[:, :, 1:4], transform)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) # B N 1
end_points['knn1'] = allSF_dist
point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1)
# # 2. graph for first EdgeConv with transform(x,y,z), SF, distance, minSF
point_cloud_all = tf.concat(axis=2, values=[point_cloud_SF,
point_cloud_transform,
point_cloud[:, :, 4:para.dim]])
adj_matrix = tf_util.pairwise_distance(point_cloud_all) # B N C=6
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(point_cloud_all, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn1', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net1 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn2'] = allSF_dist
# # 3. graph for second EdgeConv with C = 64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn2', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net2 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn3'] = allSF_dist
# # 4. graph for third EdgeConv with C = 64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn3', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net3 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn4'] = allSF_dist
# # 5. graph for fourth EdgeConv with C = 64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 128, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn4', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net4 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn5'] = allSF_dist
# # 6. MLP for all concatenate features 64+64+64+128
net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='agg', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=1, keepdims=True) # maxpooling B N C=1024 => B 1 1024
# # 7. MLP on global point cloud vector B 1 1024
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud,
up_ratio,
is_training,
bradius=1.0,
knn=30,
scope='generator',
weight_decay=0.0,
bn_decay=None,
bn=True,
fd=64,
fD=1024):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc:
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_point_cloud = tf.expand_dims(point_cloud, -1)
adj = tf_util.pairwise_distance(point_cloud)
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(input_point_cloud,
nn_idx=nn_idx,
k=knn)
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_point_cloud = tf.expand_dims(point_cloud_transformed, -1)
adj = tf_util.pairwise_distance(point_cloud_transformed)
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(input_point_cloud,
nn_idx=nn_idx,
k=knn)
out1 = tf_util.conv2d(edge_feature,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv1',
bn_decay=bn_decay)
out2 = tf_util.conv2d(out1,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv2',
bn_decay=bn_decay)
net_max_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)
net_mean_1 = tf.reduce_mean(out2, axis=-2, keep_dims=True)
out3 = tf_util.conv2d(tf.concat([net_max_1, net_mean_1], axis=-1),
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv3',
bn_decay=bn_decay)
adj = tf_util.pairwise_distance(tf.squeeze(out3, axis=-2))
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(out3, nn_idx=nn_idx, k=knn)
out4 = tf_util.conv2d(edge_feature,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv4',
bn_decay=bn_decay)
net_max_2 = tf.reduce_max(out4, axis=-2, keep_dims=True)
net_mean_2 = tf.reduce_mean(out4, axis=-2, keep_dims=True)
out5 = tf_util.conv2d(tf.concat([net_max_2, net_mean_2], axis=-1),
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv5',
bn_decay=bn_decay)
adj = tf_util.pairwise_distance(tf.squeeze(out5, axis=-2))
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(out5, nn_idx=nn_idx, k=knn)
out6 = tf_util.conv2d(edge_feature,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv6',
bn_decay=bn_decay)
net_max_3 = tf.reduce_max(out6, axis=-2, keep_dims=True)
net_mean_3 = tf.reduce_mean(out6, axis=-2, keep_dims=True)
out7 = tf_util.conv2d(tf.concat([net_max_3, net_mean_3], axis=-1),
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv7',
bn_decay=bn_decay)
out8 = tf_util.conv2d(tf.concat([out3, out5, out7], axis=-1),
fD, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='dgcnn_conv8',
bn_decay=bn_decay)
out_max = tf_util.max_pool2d(out8, [num_point, 1],
padding='VALID',
scope='maxpool')
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat_unweight = tf.concat(axis=3,
values=[
expand, net_max_1, net_mean_1, out3,
net_max_2, net_mean_2, out5, net_max_3,
net_mean_3, out7, out8
])
feat_list = [
"expand", "net_max_1", "net_mean_1", "out3", "net_max_2",
"net_mean_2", "out5", "net_max_3", "net_mean_3", "out7", "out8"
]
out_attention = tf_util.conv2d(concat_unweight,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='attention_conv1',
weight_decay=weight_decay)
out_attention = tf_util.conv2d(out_attention,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='attention_conv2',
weight_decay=weight_decay)
out_attention = tf_util.conv2d(out_attention,
len(feat_list), [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='attention_conv3',
weight_decay=weight_decay)
out_attention = tf_util.max_pool2d(out_attention, [num_point, 1],
padding='VALID',
scope='attention_maxpool')
out_attention = tf.nn.softmax(out_attention)
tmp_attention = tf.squeeze(out_attention)
for i in range(len(feat_list)):
tmp1 = tf.slice(out_attention, [0, 0, 0, i], [1, 1, 1, 1])
exec('dim = %s.get_shape()[-1].value' % feat_list[i])
tmp2 = tf.tile(tmp1, [1, 1, 1, dim])
if i == 0:
attention_weight = tmp2
else:
attention_weight = tf.concat([attention_weight, tmp2], axis=-1)
attention_weight = tf.tile(attention_weight, [1, num_point, 1, 1])
concat = tf.multiply(concat_unweight, attention_weight)
concat = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='concat_conv',
weight_decay=weight_decay)
concat = tf_util.dropout(concat,
keep_prob=0.6,
is_training=is_training,
scope='dg1')
with tf.variable_scope('uv_predict'):
uv_2d = tf_util.conv2d(concat,
up_ratio * 2, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
activation_fn=None,
bn=None,
is_training=is_training,
scope='uv_conv1',
weight_decay=weight_decay)
uv_2d = tf.reshape(uv_2d, [batch_size, num_point, up_ratio, 2])
uv_2d = tf.concat(
[uv_2d, tf.zeros([batch_size, num_point, up_ratio, 1])],
axis=-1)
with tf.variable_scope('T_predict'):
affine_T = tf_util.conv2d(concat,
9, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
activation_fn=None,
bn=None,
is_training=is_training,
scope='patch_conv1',
weight_decay=weight_decay)
affine_T = tf.reshape(affine_T, [batch_size, num_point, 3, 3])
uv_3d = tf.matmul(uv_2d, affine_T)
uv_3d = uv_3d + tf.tile(tf.expand_dims(point_cloud, axis=-2),
[1, 1, up_ratio, 1])
uv_3d = tf.transpose(uv_3d, perm=[0, 2, 1, 3])
uv_3d = tf.reshape(uv_3d, [batch_size, num_point * up_ratio, -1])
with tf.variable_scope('normal_predict'):
dense_normal_offset = tf_util.conv2d(concat,
up_ratio * 3, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
activation_fn=None,
bn=None,
is_training=is_training,
scope='normal_offset_conv1',
weight_decay=weight_decay)
dense_normal_offset = tf.reshape(
dense_normal_offset, [batch_size, num_point, up_ratio, 3])
sparse_normal = tf.convert_to_tensor([0, 0, 1], dtype=tf.float32)
sparse_normal = tf.expand_dims(sparse_normal, axis=0)
sparse_normal = tf.expand_dims(sparse_normal, axis=0)
sparse_normal = tf.expand_dims(sparse_normal, axis=0)
sparse_normal = tf.tile(sparse_normal,
[batch_size, num_point, 1, 1])
sparse_normal = tf.matmul(sparse_normal, affine_T)
sparse_normal = tf.nn.l2_normalize(sparse_normal, axis=-1)
dense_normal = tf.tile(sparse_normal,
[1, 1, up_ratio, 1]) + dense_normal_offset
dense_normal = tf.nn.l2_normalize(dense_normal, axis=-1)
dense_normal = tf.transpose(dense_normal, perm=[0, 2, 1, 3])
dense_normal = tf.reshape(dense_normal,
[batch_size, num_point * up_ratio, -1])
with tf.variable_scope('up_layer'):
if not np.isscalar(bradius):
bradius_expand = tf.expand_dims(tf.expand_dims(bradius,
axis=-1),
axis=-1)
else:
bradius_expand = bradius
bradius_expand = bradius_expand
grid = tf.expand_dims(uv_3d * bradius_expand, axis=2)
concat_up = tf.tile(concat, (1, up_ratio, 1, 1))
concat_up = tf.concat([concat_up, grid], axis=-1)
concat_up = tf_util.conv2d(concat_up,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='up_layer1',
bn_decay=bn_decay,
weight_decay=weight_decay)
concat_up = tf_util.dropout(concat_up,
keep_prob=0.6,
is_training=is_training,
scope='up_dg1')
concat_up = tf_util.conv2d(concat_up,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='up_layer2',
bn_decay=bn_decay,
weight_decay=weight_decay)
concat_up = tf_util.dropout(concat_up,
keep_prob=0.6,
is_training=is_training,
scope='up_dg2')
# get xyz
coord_z = tf_util.conv2d(concat_up,
1, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
is_training=is_training,
scope='fc_layer',
weight_decay=weight_decay)
coord_z = tf.reshape(coord_z, [batch_size, up_ratio, num_point, 1])
coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3])
coord_z = tf.concat(
[tf.zeros_like(coord_z),
tf.zeros_like(coord_z), coord_z], axis=-1)
coord_z = tf.matmul(coord_z, affine_T)
coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3])
coord_z = tf.reshape(coord_z, [batch_size, num_point * up_ratio, -1])
coord = uv_3d + coord_z
return coord, dense_normal, tf.squeeze(sparse_normal, [2])
def get_model(point_cloud,
is_training,
normals,
use_local_frame=True,
add_normals=False,
bn=True,
bn_decay=None,
use_xavier=True,
align_pointclouds=False,
drop_prob=0.5,
n_classes=1,
k=20):
""" Part segmentation DGCNN, input is BxNxnFeatures, output BxnClasses """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
# Input xyz coordinates
input_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
# Add tangent vectors and normals for local frames calculation
local_frame_data = None
if use_local_frame:
tangent_vec1 = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3],
name="tangent_vec1")
tangent_vec2 = tf.slice(point_cloud, [0, 0, 6], [-1, -1, 3],
name="tangent_vec2")
local_frame_data = (tangent_vec1, tangent_vec2, normals)
# Point clouds global alignment
if align_pointclouds:
# Calculate pairwise distance on global coordinates and find k-nn's for each point
adj = tf_util.pairwise_distance(input_xyz)
nn_idx = tf_util.knn(adj, k=k)
input_xyz = tf.expand_dims(input_xyz, -1)
edge_feature = tf_util.get_edge_feature(input_xyz, nn_idx=nn_idx, k=k)
with tf.variable_scope('transform_net_global') as sc:
global_transform = global_spatial_transformer(
point_cloud=edge_feature,
is_training=is_training,
bn=bn,
bn_decay=bn_decay,
is_dist=True)
input_xyz = tf.matmul(tf.squeeze(input_xyz, axis=-1), global_transform)
if add_normals:
if input_xyz.shape.ndims == 4:
input_xyz = tf.squeeze(input_xyz, axis=-1)
input_xyz = tf.concat([input_xyz, normals], axis=-1)
input_image = tf.expand_dims(input_xyz, -1)
adj = tf_util.pairwise_distance(input_xyz)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(input_image,
nn_idx=nn_idx,
k=k,
use_local_frame=use_local_frame,
local_frame_data=local_frame_data,
add_normals=add_normals)
# EdgeConv layer 1 {64, 64}
out1 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv1',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
out2 = tf_util.conv2d(out1,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv2',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
net_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)
# EdgeConv layer 2 {64, 64}
adj = tf_util.pairwise_distance(net_1)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k)
out3 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv3',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
out4 = tf_util.conv2d(out3,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv4',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
net_2 = tf.reduce_max(out4, axis=-2, keep_dims=True)
# EdgeConv layer 3 {64}
adj = tf_util.pairwise_distance(net_2)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k)
out5 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv5',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
net_3 = tf.reduce_max(out5, axis=-2, keep_dims=True)
# [EdgeConv1, EdgeConv2, EdgeConv3] -> MLP {64}
out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv7',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
out_max = tf_util.max_pool2d(out7, [num_point, 1],
padding='VALID',
scope='maxpool')
expand = tf.tile(out_max, [1, num_point, 1, 1])
# Concat [global_feature, EdgeConv1, EdgeConv2, EdgeConv3]
concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3])
# FC layer - MLP{256, 256, 128, n_classes}
net2 = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=bn,
is_training=is_training,
scope='seg/conv1',
is_dist=True,
use_xavier=use_xavier)
net2 = tf_util.dropout(net2,
keep_prob=1 - drop_prob,
is_training=is_training,
scope='seg/dp1')
net2 = tf_util.conv2d(net2,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=bn,
is_training=is_training,
scope='seg/conv2',
is_dist=True,
use_xavier=use_xavier)
net2 = tf_util.dropout(net2,
keep_prob=1 - drop_prob,
is_training=is_training,
scope='seg/dp2')
net2 = tf_util.conv2d(net2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=bn,
is_training=is_training,
scope='seg/conv3',
is_dist=True,
use_xavier=use_xavier)
net2 = tf_util.conv2d(net2,
n_classes, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
scope='seg/conv4',
is_dist=False,
use_xavier=use_xavier)
net2 = tf.reshape(net2, [batch_size, num_point, n_classes])
return net2
def residual_attn_block(point_cloud,
mv_fc,
k=20,
get_mask=False,
C_out=64,
C_attn=256,
is_training=True,
bn_decay=None,
scope='attn1_'):
"""
:param point_cloud: (N, P, 1, C_in)
:param mv_fc: (N, C(1024))
:param is_training: training state
:param k: k neighbors
:param C_out: output channel
:param bn_decay: bn decay
:param scope: scope name
:return: (N, P, 1, C_out)
"""
point_cloud_sq = tf.squeeze(point_cloud)
batch_size = point_cloud_sq.shape[0].value
num_points = point_cloud_sq.shape[1].value
num_dims_in = point_cloud_sq.shape[2].value
adj_matrix = tf_util.pairwise_distance(point_cloud)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k)
res1 = tf_util.conv2d(edge_feature,
C_out, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope=scope + '_res1',
bn_decay=bn_decay)
res1 = tf.reduce_max(res1, axis=-2, keep_dims=True)
# res2 = tf_util.conv2d(point_cloud, C_attn, [1,1],
# padding='VALID', stride=[1, 1],
# bn=True, is_training=is_training,
# scope=scope+'res2_conv1', bn_decay=bn_decay)
# res2_global = tf_util.max_pool2d(res2, [num_points, 1], padding='VALID', scope=scope+'maxpool')
# print (res2_global.get_shape())
res2_global = tf.expand_dims(tf.expand_dims(mv_fc, axis=1), axis=1)
res2_global = tf.tile(res2_global, [1, num_points, 1, 1])
# print (res2_global.get_shape())
res2_concat = tf.concat([res2_global, point_cloud], axis=-1)
res2_out = tf_util.conv2d(res2_concat,
C_out, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope=scope + '_res2',
bn_decay=bn_decay)
res2_mask = tf.nn.sigmoid(tf.log(res2_out))
# print (res2_mask.get_shape())
res2_attn = tf.multiply(res2_mask, res1)
# print (res2_attn.get_shape())
net = tf.add(res2_attn, res1)
# print (net.get_shape())
if get_mask:
return net, res2_mask
else:
return net
def multi_modal(point_cloud,
view_images,
is_training=False,
bn_decay=None,
n_classes=40,
get_ft=False,
get_mask=False):
"""
:param point_cloud: (B, N, 3)
:param view_images: (B, V, W, H, C)
:param is_training: is_training for dropout and bn
:param bn_decay: bn_decay
:param n_classes: 40
:return: multi-modal logit and mvcnn logit
"""
fc6_b = MVCNN.inference_multiview(view_images,
n_classes,
is_training=is_training)
mv_global = tf_util.fully_connected(fc6_b,
1024,
bn=True,
is_training=is_training,
scope='pc_mv_t',
bn_decay=bn_decay)
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
k = 20
point_cloud_transformed = trans_net(point_cloud,
k=k,
is_training=is_training,
bn_decay=bn_decay,
scope='pc_transform_net1')
adj_matrix = tf_util.pairwise_distance(point_cloud_transformed)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(point_cloud_transformed,
nn_idx=nn_idx,
k=k)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='pc1',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net1 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='pc2',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net2 = net
net = residual_attn_block(net,
mv_global,
k=k,
C_out=64,
get_mask=get_mask,
C_attn=256,
is_training=is_training,
bn_decay=bn_decay,
scope='pc_attn_1')
if get_mask:
net3, mask1 = net
net = net3
else:
net3 = net
net = residual_attn_block(net,
mv_global,
k=k,
C_out=128,
get_mask=get_mask,
C_attn=256,
is_training=is_training,
bn_decay=bn_decay,
scope='pc_attn_2')
if get_mask:
net4, mask2 = net
net = net3
else:
net4 = net
net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='pc_agg',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=1, keep_dims=True)
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf.concat([net, mv_global], axis=-1)
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='pc_fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='pc_dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='pc_fc2',
bn_decay=bn_decay)
ft = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='pc_dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='pc_fc3')
if get_ft:
return net, ft
elif get_mask:
return net, mask1, mask2
else:
return net