def create_decoder(self, features):
"""fully connected layers for classification with dropout"""
with tf.variable_scope('decoder_cls', reuse=tf.AUTO_REUSE):
features = fully_connected(features,
512,
bn=True,
scope='fc1',
is_training=self.is_training)
features = dropout(features,
keep_prob=0.7,
scope='dp1',
is_training=self.is_training)
features = fully_connected(features,
256,
bn=True,
scope='fc2',
is_training=self.is_training)
features = dropout(features,
keep_prob=0.7,
scope='dp2',
is_training=self.is_training)
pred = fully_connected(features,
NUM_CLASSES,
activation_fn=None,
scope='fc3',
is_training=self.is_training)
return pred
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image, 64, [1,3],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = tf_util.conv2d(net_transformed, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
"""
Classification PointNetwork
:param point_cloud: point cloud BxNx3
:param is_training: training flag
:param bn_decay: decay flag
:return: output model Bx40
"""
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
input_image = tf.expand_dims(point_cloud, -1)
# Point functions (MPL implemented as conv2d)
net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
# Symetric function: Max Pooling
net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool')
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def create_decoder(self): net = conv2d(self.cat_feats, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv6') net = conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv7') net = dropout(net, keep_prob=0.7, is_training=self.is_training, scope='dp1') net = conv2d(net, 13, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv8') net = tf.squeeze(net, [2]) return net
def create_decoder(self, features):
"""fully connected layers for classification with dropout"""
with tf.variable_scope('decoder_cls', reuse=tf.AUTO_REUSE):
# self.linear1 = nn.Linear(args.emb_dims*2, 512, bias=False)
features = tf_util.fully_connected(features,
512,
bn=True,
bias=False,
activation_fn=tf.nn.leaky_relu,
scope='linear1',
is_training=self.is_training)
features = tf_util.dropout(features,
keep_prob=0.5,
scope='dp1',
is_training=self.is_training)
# self.linear2 = nn.Linear(512, 256)
features = tf_util.fully_connected(features,
256,
bn=True,
bias=True,
activation_fn=tf.nn.leaky_relu,
scope='linear2',
is_training=self.is_training)
features = tf_util.dropout(features,
keep_prob=0.5,
scope='dp2',
is_training=self.is_training)
# self.linear3 = nn.Linear(256, output_channels)
pred = tf_util.fully_connected(features,
NUM_CLASSES,
bn=False,
bias=True,
activation_fn=None,
scope='linear3',
is_training=self.is_training)
return pred
def get_model(point_cloud, is_training, bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -1)
# CONV
net = tf_util.conv2d(input_image, 64, [1,9], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay)
points_feat1 = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay)
# MAX
pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point,1], padding='VALID', scope='maxpool1')
# FC
pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1])
pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
pc_feat1 = tf_util.fully_connected(pc_feat1, 128, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
print(pc_feat1)
# CONCAT
pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1])
print(points_feat1)
print(pc_feat1_expand)
points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand])
print(points_feat1_concat)
# CONV
net = tf_util.conv2d(points_feat1_concat, 512, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv6')
net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv7')
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1')
net = tf_util.conv2d(net, 13, [1,1], padding='VALID', stride=[1,1],
activation_fn=None, scope='conv8')
net = tf.squeeze(net, [2])
return net
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = tf.shape(point_cloud)[0]
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4, 0.8], [64, 128],
[[128, 128, 256], [128, 196, 256]], is_training, bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None,
mlp=[256, 512, 1024], mlp2=None, group_all=True,
is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay,
scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points], axis=-1),
l1_points, [128, 128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def inference_multiview_all(images, n_classes=40, is_training=False):
"""
:param images: (N, V, W, H, C)
:param is_training: is_training
:param bn_decay: bn_decay
:return: fc logits
"""
# batch_size = images.get_shape().as_list()[0]
# n_views = images.get_shape().as_list()[1]
# weight = images.get_shape().as_list()[2]
# height = images.get_shape().as_list()[3]
# dims = images.get_shape().as_list()[4]
batch_size = images.shape[0].value
n_views = images.shape[1].value
weight = images.shape[2].value
height = images.shape[3].value
dims = images.shape[4].value
# Get images (N*V, W, H, C)
images = tf.reshape(images, [batch_size * n_views, weight, height, dims])
reuse = False
conv1 = _conv('conv1',
images, [11, 11, 3, 96], [1, 4, 4, 1],
'VALID',
reuse=reuse)
lrn1 = None
pool1 = _maxpool('pool1',
conv1,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID')
conv2 = _conv('conv2', pool1, [5, 5, 96, 256], group=2, reuse=reuse)
lrn2 = None
pool2 = _maxpool('pool2',
conv2,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID')
conv3 = _conv('conv3', pool2, [3, 3, 256, 384], reuse=reuse)
conv4 = _conv('conv4', conv3, [3, 3, 384, 384], group=2, reuse=reuse)
conv5 = _conv('conv5', conv4, [3, 3, 384, 256], group=2, reuse=reuse)
pool5 = _maxpool('pool5',
conv5,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID')
hwc = np.prod(pool5.get_shape().as_list()[1:])
flatten = tf.reshape(pool5, [batch_size * n_views, hwc])
view_pooling = tf.reshape(flatten, [batch_size, n_views, hwc])
view_pooling = tf.reduce_max(view_pooling, axis=1)
# print 'pool5_vp', view_pooling.get_shape().as_list()
fc6_b = _fc('fc6', view_pooling, 4096)
fc6 = tf_util.dropout(fc6_b, is_training, scope='dp6', keep_prob=0.5)
fc7 = _fc('fc7', fc6, 4096)
fc7 = tf_util.dropout(fc7, is_training, scope='dp7', keep_prob=0.5)
fc8 = _fc('fc8', fc7, n_classes)
return fc8
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Set abstraction layers
# Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4).
# So we only use NCHW for layer 1 until this issue can be resolved.
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=512,
radius=0.2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=128,
radius=0.4,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.layers.flatten(l3_points)
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, num_class, hyperparams, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
if hyperparams['use_color'] or hyperparams['use_z_feature']:
feature_size = 3 * int(hyperparams['use_color']) + int(
hyperparams['use_z_feature'])
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, feature_size])
else:
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(
l0_xyz,
l0_points,
npoint=hyperparams['l1_npoint'],
radius=hyperparams['l1_radius'],
nsample=hyperparams['l1_nsample'],
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(
l1_xyz,
l1_points,
npoint=hyperparams['l2_npoint'],
radius=hyperparams['l2_radius'],
nsample=hyperparams['l2_nsample'],
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(
l2_xyz,
l2_points,
npoint=hyperparams['l3_npoint'],
radius=hyperparams['l3_radius'],
nsample=hyperparams['l3_nsample'],
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(
l3_xyz,
l3_points,
npoint=hyperparams['l4_npoint'],
radius=hyperparams['l4_radius'],
nsample=hyperparams['l4_nsample'],
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model_other(point_cloud, is_training, bn_decay=None):
""" Classification DGCNN, input is BxNxC, output BxCls
B batch size
N number of points per pointcloud
C input channel: eg. x,y,z,SF,distance, minSF...
Cls output class number
"""
batch_size = point_cloud.get_shape()[0] # .value
end_points = {}
# get MinSF index
minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1))
# # 1. graph for transform net with only x,y,z
adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:4]) # B N C=3 => B N N
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(point_cloud[:, :, 1:4], nn_idx=nn_idx, k=para.k)
with tf.compat.v1.variable_scope('transform_net1') as sc:
transform = input_transform_net_dgcnn(edge_feature, is_training, bn_decay, K=3)
point_cloud_transform = tf.matmul(point_cloud[:, :, 1:4], transform)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) # B N 1
end_points['knn1'] = allSF_dist
point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1)
# # 2. graph for first EdgeConv with transform(x,y,z), SF, distance, minSF
point_cloud_all = tf.concat(axis=2, values=[point_cloud_SF,
point_cloud_transform,
point_cloud[:, :, 4:para.dim]])
adj_matrix = tf_util.pairwise_distance(point_cloud_all) # B N C=6
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(point_cloud_all, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn1', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net1 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn2'] = allSF_dist
# # 3. graph for second EdgeConv with C = 64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn2', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net2 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn3'] = allSF_dist
# # 4. graph for third EdgeConv with C = 64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn3', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net3 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn4'] = allSF_dist
# # 5. graph for fourth EdgeConv with C = 64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=para.k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
net = tf_util.conv2d(edge_feature, 128, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='dgcnn4', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
net4 = net
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn5'] = allSF_dist
# # 6. MLP for all concatenate features 64+64+64+128
net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='agg', bn_decay=bn_decay)
net = tf.reduce_max(input_tensor=net, axis=1, keepdims=True) # maxpooling B N C=1024 => B 1 1024
# # 7. MLP on global point cloud vector B 1 1024
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud,
input_label,
is_training,
cat_num,
part_num,
batch_size,
num_point,
weight_decay,
bn_decay=None):
"""
Get ConvNet baseline model.
:param point_cloud:
:param input_label:
:param is_training:
:param cat_num:
:param part_num:
:param batch_size:
:param num_point:
:param weight_decay:
:param bn_decay:
:return:
"""
end_points = {}
with tf.variable_scope('transform_net1') as sc:
K = 3
transform = get_transform(point_cloud, is_training, bn_decay, k=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
out1 = tf_util.conv2d(input_image,
64, [1, K],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
out2 = tf_util.conv2d(out1,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
out3 = tf_util.conv2d(out2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
K = 128
transform = get_transform_K(out3, is_training, bn_decay, K)
end_points['transform'] = transform
squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128])
net_transformed = tf.matmul(squeezed_out3, transform)
net_transformed = tf.expand_dims(net_transformed, [2])
out4 = tf_util.conv2d(net_transformed,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
out5 = tf_util.conv2d(out4,
2048, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
out_max = tf_util.max_pool2d(out5, [num_point, 1],
padding='VALID',
scope='maxpool')
# classification network
net = tf.reshape(out_max, [batch_size, -1])
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='cla/fc1',
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='cla/fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='cla/dp1')
net = tf_util.fully_connected(net,
cat_num,
activation_fn=None,
scope='cla/fc3')
# segmentation network
one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num])
out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand])
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5])
net2 = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv1',
weight_decay=weight_decay)
net2 = tf_util.dropout(net2,
keep_prob=0.8,
is_training=is_training,
scope='seg/dp1')
net2 = tf_util.conv2d(net2,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv2',
weight_decay=weight_decay)
net2 = tf_util.dropout(net2,
keep_prob=0.8,
is_training=is_training,
scope='seg/dp2')
net2 = tf_util.conv2d(net2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv3',
weight_decay=weight_decay)
net2 = tf_util.conv2d(net2,
part_num, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
scope='seg/conv4',
weight_decay=weight_decay)
net2 = tf.reshape(net2, [batch_size, num_point, part_num])
return net, net2, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet++, input is BxNxF, output Bxnum_class """
end_points = {}
# COG
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
# Features
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, -1])
end_points['l0_xyz'] = l0_xyz
# Set Abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz,
l0_points,
npoint=4096,
radius_list=[0.5, 1, 2],
nsample_list=[8, 16, 32],
mlp_list=[[32, 32, 64],
[64, 64, 128],
[128, 128, 256]],
is_training=is_training,
pooling='max_and_avg',
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz,
l1_points,
npoint=256,
radius_list=[2, 4, 8],
nsample_list=[32, 64, 128],
mlp_list=[[128, 128, 256],
[256, 256, 512],
[256, 256, 512]],
is_training=is_training,
pooling='max_and_avg',
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points = pointnet_sa_module_msg(l2_xyz,
l2_points,
npoint=32,
radius_list=[4, 8, 16],
nsample_list=[64, 64, 128],
mlp_list=[[256, 256, 512],
[512, 512, 1024],
[512, 1024, 1024]],
is_training=is_training,
pooling='max_and_avg',
bn_decay=bn_decay,
scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [512, 512],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def multi_modal(point_cloud,
view_images,
is_training=False,
bn_decay=None,
n_classes=40,
get_ft=False,
get_mask=False):
"""
:param point_cloud: (B, N, 3)
:param view_images: (B, V, W, H, C)
:param is_training: is_training for dropout and bn
:param bn_decay: bn_decay
:param n_classes: 40
:return: multi-modal logit and mvcnn logit
"""
fc6_b = MVCNN.inference_multiview(view_images,
n_classes,
is_training=is_training)
mv_global = tf_util.fully_connected(fc6_b,
1024,
bn=True,
is_training=is_training,
scope='pc_mv_t',
bn_decay=bn_decay)
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
k = 20
point_cloud_transformed = trans_net(point_cloud,
k=k,
is_training=is_training,
bn_decay=bn_decay,
scope='pc_transform_net1')
adj_matrix = tf_util.pairwise_distance(point_cloud_transformed)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(point_cloud_transformed,
nn_idx=nn_idx,
k=k)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='pc1',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net1 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='pc2',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net2 = net
net = residual_attn_block(net,
mv_global,
k=k,
C_out=64,
get_mask=get_mask,
C_attn=256,
is_training=is_training,
bn_decay=bn_decay,
scope='pc_attn_1')
if get_mask:
net3, mask1 = net
net = net3
else:
net3 = net
net = residual_attn_block(net,
mv_global,
k=k,
C_out=128,
get_mask=get_mask,
C_attn=256,
is_training=is_training,
bn_decay=bn_decay,
scope='pc_attn_2')
if get_mask:
net4, mask2 = net
net = net3
else:
net4 = net
net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='pc_agg',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=1, keep_dims=True)
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf.concat([net, mv_global], axis=-1)
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='pc_fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='pc_dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='pc_fc2',
bn_decay=bn_decay)
ft = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='pc_dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='pc_fc3')
if get_ft:
return net, ft
elif get_mask:
return net, mask1, mask2
else:
return net
def get_model(inputs, is_training, bn_decay=None, num_class=40, FLAGS=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
point_cloud = inputs[:, :, 0:3]
if FLAGS.normal:
D = 6
points = inputs[:, :, 3:]
else:
D = 3
points = None
# --------------------------------------- STN -------------------------------------
if FLAGS.STN:
with tf.variable_scope('transform_net') as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud = tf.matmul(point_cloud, transform)
# ---------------------------------- Node Sampling --------------------------------
with tf.variable_scope('group_sampling') as sc:
KNN = FLAGS.KNN
point_cloud_sampled, nn_points, _, _ = sample_and_group(
npoint=FLAGS.node_num,
radius=0.2,
nsample=KNN,
xyz=point_cloud,
points=points,
knn=True,
use_xyz=True)
point_cloud_sampled = tf.expand_dims(point_cloud_sampled, axis=-1)
net1 = tf_util.conv2d(point_cloud_sampled,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1_1',
bn_decay=bn_decay)
net1 = tf.tile(net1, multiples=[1, 1, KNN, 1])
net1 = tf.expand_dims(net1, axis=-2)
nn_points = tf.expand_dims(nn_points, axis=-1)
net = tf_util.conv3d(nn_points,
64, [1, 1, D],
padding='VALID',
stride=[1, 1, 1],
bn=True,
is_training=is_training,
scope='conv1_2',
bn_decay=bn_decay)
concat = tf.concat(values=[net, net1], axis=-1)
net = tf_util.conv3d(concat,
128, [1, 1, 1],
padding='VALID',
stride=[1, 1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv3d(net,
128, [1, 1, 1],
padding='VALID',
stride=[1, 1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
# ---------------------- local pooling: merge local feature -------------------------
if FLAGS.local_pool == 'average':
pool_k = tf_util.avg_pool3d(net,
kernel_size=[1, KNN, 1],
stride=[1, 2, 2],
padding='VALID',
scope='pool_k')
else:
pool_k = tf_util.max_pool3d(net,
kernel_size=[1, KNN, 1],
stride=[1, 2, 2],
padding='VALID',
scope='pool_k')
net = tf.squeeze(pool_k, axis=2)
# ---------------------------------- VLAD layer --------------------------------------
net, index = VLAD(net, FLAGS, is_training, bn_decay, layer_name='VLAD')
# -------------------------------- classification ------------------------------------
with tf.name_scope('fc_layer'):
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net,
num_class,
activation_fn=None,
scope='fc3')
return net, index
def pointnet(self, point_cloud, is_training, bn=True, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bxn where n is num classes """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -1)
# Point functions (MLP implemented as conv2d)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
# MLP on global point cloud vector
net = tf.layers.flatten(net)
net = tf_util.fully_connected(net,
512,
bn=bn,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
256,
bn=bn,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
self.n_classes,
activation_fn=None,
scope='fc3')
return net
def get_model(inputs, cls_label, is_training=None, bn_decay=None, NUM_CATEGORIES=16):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = inputs.get_shape()[0].value
num_point = inputs.get_shape()[1].value
point_cloud = inputs[:, :, 0:3]
points = inputs[:, :, 3:]
with tf.variable_scope('transform_net') as sc:
transform = input_transform_net(point_cloud, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
with tf.variable_scope('group_sampling') as sc:
KNN = 16
Node_NUM = 256
point_cloud_sampled, nn_points, _, _ = sample_and_group(npoint=Node_NUM, radius=0.2, nsample=KNN,
xyz=point_cloud_transformed, points=points, knn=True, use_xyz=True)
net1 = tf_util.conv2d(tf.expand_dims(point_cloud_sampled, axis=-1), 64, [1, 3],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='conv1_1', bn_decay=bn_decay)
net1 = tf.tile(net1, multiples=[1, 1, KNN, 1])
net1 = tf.expand_dims(net1, axis=-2)
nn_points = tf.expand_dims(nn_points, axis=-1)
net = tf_util.conv3d(nn_points, 64, [1, 1, 6],
padding='VALID', stride=[1, 1, 1],
bn=True, is_training=is_training,
scope='conv1_2', bn_decay=bn_decay)
concat = tf.concat(values=[net, net1], axis=-1)
net = tf_util.conv3d(concat, 128, [1, 1, 1],
padding='VALID', stride=[1, 1, 1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
net = tf_util.conv3d(net, 128, [1, 1, 1],
padding='VALID', stride=[1, 1, 1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
pool_k = tf_util.max_pool3d(net, kernel_size=[1, KNN, 1],
stride=[1, 2, 2], padding='VALID', scope='pool_k')
net = tf.squeeze(pool_k, axis=2)
# vlad layer
vlad_out, index = VLAD_part(net, 50, is_training, bn_decay, layer_name='VLAD')
vlad_out = tf_util.conv2d(vlad_out, 384, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='vlad_conv3', bn_decay=bn_decay)
vlad_out = tf_util.conv2d(vlad_out, 512, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='vlad_conv4', bn_decay=bn_decay)
out_max = tf.nn.max_pool(vlad_out, ksize=[1, Node_NUM, 1, 1], strides=[1, 2, 2, 1],
padding='VALID')
expand = tf.tile(out_max, multiples=[1,Node_NUM,1,1])
concat = tf.concat([expand, vlad_out, net], axis=-1)
concat = tf_util.conv2d(concat, 512, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
concat = tf_util.conv2d(concat, 256, [1, 1],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
concat = tf.squeeze(concat, axis=2)
# segmentation network
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(xyz1=point_cloud_transformed, xyz2=point_cloud_sampled, points1=tf.concat([cls_label_one_hot, point_cloud_transformed, points], axis=-1),
points2=concat, mlp=[128, 128], is_training=is_training, bn_decay=bn_decay, scope='layer6')
l0_points = tf.expand_dims(l0_points, axis=2)
net = tf_util.conv2d(l0_points, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv2d(net, 50, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='fc2')
net = tf.squeeze(net, axis=2)
return net, index
def get_model_other(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3 and BxNx1, output Bx4 """
batch_size = point_cloud.get_shape()[0] # .value
num_point = point_cloud.get_shape()[1] # .value
end_points = {}
# transform net for input x,y,z
with tf.compat.v1.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud[:, :, 1:4],
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud[:, :, 1:4], transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
# First MLP layers
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
# transform net for the features
with tf.compat.v1.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
# point_feat = tf.expand_dims(net_transformed, [2])
# add the additional features to the second MLP layers
# point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0:1], [2])
concat_other = tf.concat(axis=2,
values=[
point_cloud[:, :, 0:1], net_transformed,
point_cloud[:, :, 4:para.dim]
])
concat_other = tf.expand_dims(concat_other, [2])
# second MLP layers
net = tf_util.conv2d(
concat_other,
64,
[1, 1], # 64 is the output #neuron
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 4, activation_fn=None, scope='fc3')
return net, end_points
def pct_model(point_cloud, is_training, bn_decay=None):
"""
:param point_cloud:
:param is_training:
:param bn_decay:
:return:
"""
# point_cloud -> [batch_size, num_point, 3]
batch_size = point_cloud.get_shape()[0].value
point_dim = point_cloud.get_shape()[2].value
"""Input Embedding module"""
# [batch_size, num_point, 64]
x = tf_util.conv1d(point_cloud, 64, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope='conv0', bn_decay=bn_decay)
x = tf_util.conv1d(x, 64, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
""" Sample and Group """
new_xyz, new_feature, _, _ = pointnet_util.sample_and_group(
npoint=512, radius=0.15,
nsample=32, xyz=point_cloud,
points=x, knn=True, use_xyz=True)
# print(new_xyz.shape)
# print("new_feature.shape", new_feature.shape)
feature_0 = local_op(new_feature, out_dim=128, scope="SG1",
bn_decay=bn_decay, is_training=is_training)
new_xyz, new_feature, _, _ = pointnet_util.sample_and_group(
npoint=256, radius=0.2,
nsample=32, xyz=new_xyz,
points=feature_0,
knn=True, use_xyz=True)
# NHC
feature_1 = local_op(new_feature, out_dim=256, scope="SG2",
bn_decay=bn_decay, is_training=is_training)
#
# NHC
x = pt_last(feature_1, scope="pct_layer", out_dim=256,
bn_decay=bn_decay, is_training=is_training)
# concat in C (NHC) axis
x = tf.concat([x, feature_1], axis=-1)
x = tf_util.conv1d(x, 1024, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope='conv2', bn_decay=bn_decay, activation_fn=None)
x = tf.nn.leaky_relu(x, alpha=0.2)
x = tf.reduce_max(x, axis=1)
"""
++++++++++++++++++++++++++++++++++++++++
Decoder
++++++++++++++++++++++++++++++++++++++++
"""
x = tf_util.fully_connected(x, 512, bn=True, is_training=is_training, is_biases=False,
scope='fc1', bn_decay=bn_decay)
x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp1')
x = tf_util.fully_connected(x, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp2')
x = tf_util.fully_connected(x, 40, activation_fn=None, scope='fc3')
return x
def get_model_other(point_cloud, is_training, bn_decay=None):
"""
PointNet2 with multi-scale grouping
Semantic segmentation network that uses feature propogation layers
"""
batch_size = point_cloud.get_shape()[0] # .value
end_points = {}
l0_xyz = point_cloud[:, :, 1:4]
l0_points = tf.concat(
axis=2, values=[point_cloud[:, :, 0:1], point_cloud[:, :, 4:para.dim]])
# Set abstraction layers
# input B 1024 1 3 => 64+128+128 = 320 max pooling in small group n = 16 32 128
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1') # , use_nchw=True
# input B 512 320 => 128+256+256 = 640 max pooling in small group n = 32 64 128
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2')
# input B 128 640 => 1024, max pooling in all pointcloud = 128
# MLP layer to gather 3 scale features
_, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# input B 1 1024
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net,
para.outputClassN,
activation_fn=None,
scope='fc3')
return net, end_points
def get_model_other(point_cloud, is_training, bn_decay=None):
"""
DesnsePoint with 2 PPools + 3 PConvs + 1 global pooling narrowness k = 24; group number g = 2
"""
batch_size = point_cloud.get_shape()[0] # .value
end_points = {}
# concatenate all features together SF,x,y,z, distance minSF
point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1)
l0_xyz = point_cloud[:, :, 1:4]
l0_points = tf.concat(
axis=2, values=[point_cloud_SF, point_cloud[:, :, 4:para.dim]])
# first stage: 1 PPool, 3 EnhancedPConv
all_xyz, all_points = densepoint_module(l0_xyz,
l0_points,
is_training,
bn_decay,
npoint=512,
radius=0.25,
nsample=64,
mlp=para.k_add * 4,
scope='PPool1',
ppool=True)
# second stage: 2 PPool, 3 EnhancedPConv
all_xyz, all_points = densepoint_module(all_xyz,
all_points,
is_training,
bn_decay,
npoint=256,
radius=0.27,
nsample=64,
mlp=para.k_add * 4 - 3,
scope='PPool2',
ppool=True)
for i in range(4): # B 128 1 93 -> 24
all_xyz, all_points = densepoint_module(all_xyz,
all_points,
is_training,
bn_decay,
npoint=256,
radius=0.32,
nsample=16,
mlp=para.k_add * 4,
group_num=para.group_num,
scope=f'PConv1_{i + 1}')
# # second stage: 2 PPool, 3 EnhancedPConv
# all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay,
# npoint=128, radius=0.32, nsample=64, mlp=para.k_add * 4 - 3,
# scope='PPool3', ppool=True)
#
# for i in range(4): # B 128 1 93 -> 24
# all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay,
# npoint=128, radius=0.39, nsample=16, mlp=para.k_add * 4,
# group_num=para.group_num,
# scope=f'PConv2_{i + 1}')
l3_points = densepoint_module(all_xyz,
all_points,
is_training,
bn_decay,
mlp=512,
scope='GloPool')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net,
para.outputClassN,
activation_fn=None,
scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, part_num, batch_size, \
num_point, weight_decay, bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
end_points = {}
with tf.variable_scope('transform_net1') as sc:
K = 3
transform = get_transform(point_cloud, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
out1 = tf_util.conv2d(input_image,
64, [1, K],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
out2 = tf_util.conv2d(out1,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
out3 = tf_util.conv2d(out2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
K = 128
transform = get_transform_K(out3, is_training, bn_decay, K)
end_points['transform'] = transform
squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128])
net_transformed = tf.matmul(squeezed_out3, transform)
net_transformed = tf.expand_dims(net_transformed, [2])
out4 = tf_util.conv2d(net_transformed,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
out5 = tf_util.conv2d(out4,
2048, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
out_max = tf_util.max_pool2d(out5, [num_point, 1],
padding='VALID',
scope='maxpool')
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5])
net = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv1',
weight_decay=weight_decay)
net = tf_util.dropout(net,
keep_prob=0.8,
is_training=is_training,
scope='seg/dp1')
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv2',
weight_decay=weight_decay)
net = tf_util.dropout(net,
keep_prob=0.8,
is_training=is_training,
scope='seg/dp2')
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv3',
weight_decay=weight_decay)
net = tf_util.conv2d(net,
part_num, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
scope='seg/conv4',
weight_decay=weight_decay)
net = tf.reshape(net, [batch_size, num_point, part_num])
return net, end_points
def get_model(point_cloud,
is_training,
normals,
use_local_frame=True,
add_normals=False,
bn=True,
bn_decay=None,
use_xavier=True,
align_pointclouds=False,
drop_prob=0.5,
n_classes=1,
k=20):
""" Part segmentation DGCNN, input is BxNxnFeatures, output BxnClasses """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
# Input xyz coordinates
input_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
# Add tangent vectors and normals for local frames calculation
local_frame_data = None
if use_local_frame:
tangent_vec1 = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3],
name="tangent_vec1")
tangent_vec2 = tf.slice(point_cloud, [0, 0, 6], [-1, -1, 3],
name="tangent_vec2")
local_frame_data = (tangent_vec1, tangent_vec2, normals)
# Point clouds global alignment
if align_pointclouds:
# Calculate pairwise distance on global coordinates and find k-nn's for each point
adj = tf_util.pairwise_distance(input_xyz)
nn_idx = tf_util.knn(adj, k=k)
input_xyz = tf.expand_dims(input_xyz, -1)
edge_feature = tf_util.get_edge_feature(input_xyz, nn_idx=nn_idx, k=k)
with tf.variable_scope('transform_net_global') as sc:
global_transform = global_spatial_transformer(
point_cloud=edge_feature,
is_training=is_training,
bn=bn,
bn_decay=bn_decay,
is_dist=True)
input_xyz = tf.matmul(tf.squeeze(input_xyz, axis=-1), global_transform)
if add_normals:
if input_xyz.shape.ndims == 4:
input_xyz = tf.squeeze(input_xyz, axis=-1)
input_xyz = tf.concat([input_xyz, normals], axis=-1)
input_image = tf.expand_dims(input_xyz, -1)
adj = tf_util.pairwise_distance(input_xyz)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(input_image,
nn_idx=nn_idx,
k=k,
use_local_frame=use_local_frame,
local_frame_data=local_frame_data,
add_normals=add_normals)
# EdgeConv layer 1 {64, 64}
out1 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv1',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
out2 = tf_util.conv2d(out1,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv2',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
net_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)
# EdgeConv layer 2 {64, 64}
adj = tf_util.pairwise_distance(net_1)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k)
out3 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv3',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
out4 = tf_util.conv2d(out3,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv4',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
net_2 = tf.reduce_max(out4, axis=-2, keep_dims=True)
# EdgeConv layer 3 {64}
adj = tf_util.pairwise_distance(net_2)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k)
out5 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv5',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
net_3 = tf.reduce_max(out5, axis=-2, keep_dims=True)
# [EdgeConv1, EdgeConv2, EdgeConv3] -> MLP {64}
out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='adj_conv7',
bn_decay=bn_decay,
is_dist=True,
use_xavier=use_xavier)
out_max = tf_util.max_pool2d(out7, [num_point, 1],
padding='VALID',
scope='maxpool')
expand = tf.tile(out_max, [1, num_point, 1, 1])
# Concat [global_feature, EdgeConv1, EdgeConv2, EdgeConv3]
concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3])
# FC layer - MLP{256, 256, 128, n_classes}
net2 = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=bn,
is_training=is_training,
scope='seg/conv1',
is_dist=True,
use_xavier=use_xavier)
net2 = tf_util.dropout(net2,
keep_prob=1 - drop_prob,
is_training=is_training,
scope='seg/dp1')
net2 = tf_util.conv2d(net2,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=bn,
is_training=is_training,
scope='seg/conv2',
is_dist=True,
use_xavier=use_xavier)
net2 = tf_util.dropout(net2,
keep_prob=1 - drop_prob,
is_training=is_training,
scope='seg/dp2')
net2 = tf_util.conv2d(net2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=bn,
is_training=is_training,
scope='seg/conv3',
is_dist=True,
use_xavier=use_xavier)
net2 = tf_util.conv2d(net2,
n_classes, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
scope='seg/conv4',
is_dist=False,
use_xavier=use_xavier)
net2 = tf.reshape(net2, [batch_size, num_point, n_classes])
return net2
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNxF, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
input_image = tf.expand_dims(point_cloud, -1)
# CONV layers
net = tf_util.conv2d(input_image,
64, [1, 9],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
points_feat1 = tf_util.conv2d(net,
2048, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# MAX Pooling
pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point, 1],
padding='VALID',
scope='maxpool1')
# FC layers
pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1])
pc_feat1 = tf_util.fully_connected(pc_feat1,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
pc_feat1 = tf_util.fully_connected(pc_feat1,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
# CONCAT
pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]),
[1, num_point, 1, 1])
points_feat1_concat = tf.concat(axis=3,
values=[points_feat1, pc_feat1_expand])
# CONV Layers
net = tf_util.conv2d(points_feat1_concat,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv6')
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv7')
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv8')
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv9')
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv2d(net,
num_class, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
scope='conv10')
net = tf.squeeze(net, [2])
return net, end_points
def get_model(point_cloud,
up_ratio,
is_training,
bradius=1.0,
knn=30,
scope='generator',
weight_decay=0.0,
bn_decay=None,
bn=True,
fd=64,
fD=1024):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc:
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_point_cloud = tf.expand_dims(point_cloud, -1)
adj = tf_util.pairwise_distance(point_cloud)
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(input_point_cloud,
nn_idx=nn_idx,
k=knn)
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_point_cloud = tf.expand_dims(point_cloud_transformed, -1)
adj = tf_util.pairwise_distance(point_cloud_transformed)
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(input_point_cloud,
nn_idx=nn_idx,
k=knn)
out1 = tf_util.conv2d(edge_feature,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv1',
bn_decay=bn_decay)
out2 = tf_util.conv2d(out1,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv2',
bn_decay=bn_decay)
net_max_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)
net_mean_1 = tf.reduce_mean(out2, axis=-2, keep_dims=True)
out3 = tf_util.conv2d(tf.concat([net_max_1, net_mean_1], axis=-1),
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv3',
bn_decay=bn_decay)
adj = tf_util.pairwise_distance(tf.squeeze(out3, axis=-2))
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(out3, nn_idx=nn_idx, k=knn)
out4 = tf_util.conv2d(edge_feature,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv4',
bn_decay=bn_decay)
net_max_2 = tf.reduce_max(out4, axis=-2, keep_dims=True)
net_mean_2 = tf.reduce_mean(out4, axis=-2, keep_dims=True)
out5 = tf_util.conv2d(tf.concat([net_max_2, net_mean_2], axis=-1),
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv5',
bn_decay=bn_decay)
adj = tf_util.pairwise_distance(tf.squeeze(out5, axis=-2))
nn_idx = tf_util.knn(adj, k=knn)
edge_feature = tf_util.get_edge_feature(out5, nn_idx=nn_idx, k=knn)
out6 = tf_util.conv2d(edge_feature,
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv6',
bn_decay=bn_decay)
net_max_3 = tf.reduce_max(out6, axis=-2, keep_dims=True)
net_mean_3 = tf.reduce_mean(out6, axis=-2, keep_dims=True)
out7 = tf_util.conv2d(tf.concat([net_max_3, net_mean_3], axis=-1),
fd, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
weight_decay=weight_decay,
scope='dgcnn_conv7',
bn_decay=bn_decay)
out8 = tf_util.conv2d(tf.concat([out3, out5, out7], axis=-1),
fD, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='dgcnn_conv8',
bn_decay=bn_decay)
out_max = tf_util.max_pool2d(out8, [num_point, 1],
padding='VALID',
scope='maxpool')
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat_unweight = tf.concat(axis=3,
values=[
expand, net_max_1, net_mean_1, out3,
net_max_2, net_mean_2, out5, net_max_3,
net_mean_3, out7, out8
])
feat_list = [
"expand", "net_max_1", "net_mean_1", "out3", "net_max_2",
"net_mean_2", "out5", "net_max_3", "net_mean_3", "out7", "out8"
]
out_attention = tf_util.conv2d(concat_unweight,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='attention_conv1',
weight_decay=weight_decay)
out_attention = tf_util.conv2d(out_attention,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='attention_conv2',
weight_decay=weight_decay)
out_attention = tf_util.conv2d(out_attention,
len(feat_list), [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='attention_conv3',
weight_decay=weight_decay)
out_attention = tf_util.max_pool2d(out_attention, [num_point, 1],
padding='VALID',
scope='attention_maxpool')
out_attention = tf.nn.softmax(out_attention)
tmp_attention = tf.squeeze(out_attention)
for i in range(len(feat_list)):
tmp1 = tf.slice(out_attention, [0, 0, 0, i], [1, 1, 1, 1])
exec('dim = %s.get_shape()[-1].value' % feat_list[i])
tmp2 = tf.tile(tmp1, [1, 1, 1, dim])
if i == 0:
attention_weight = tmp2
else:
attention_weight = tf.concat([attention_weight, tmp2], axis=-1)
attention_weight = tf.tile(attention_weight, [1, num_point, 1, 1])
concat = tf.multiply(concat_unweight, attention_weight)
concat = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='concat_conv',
weight_decay=weight_decay)
concat = tf_util.dropout(concat,
keep_prob=0.6,
is_training=is_training,
scope='dg1')
with tf.variable_scope('uv_predict'):
uv_2d = tf_util.conv2d(concat,
up_ratio * 2, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
activation_fn=None,
bn=None,
is_training=is_training,
scope='uv_conv1',
weight_decay=weight_decay)
uv_2d = tf.reshape(uv_2d, [batch_size, num_point, up_ratio, 2])
uv_2d = tf.concat(
[uv_2d, tf.zeros([batch_size, num_point, up_ratio, 1])],
axis=-1)
with tf.variable_scope('T_predict'):
affine_T = tf_util.conv2d(concat,
9, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
activation_fn=None,
bn=None,
is_training=is_training,
scope='patch_conv1',
weight_decay=weight_decay)
affine_T = tf.reshape(affine_T, [batch_size, num_point, 3, 3])
uv_3d = tf.matmul(uv_2d, affine_T)
uv_3d = uv_3d + tf.tile(tf.expand_dims(point_cloud, axis=-2),
[1, 1, up_ratio, 1])
uv_3d = tf.transpose(uv_3d, perm=[0, 2, 1, 3])
uv_3d = tf.reshape(uv_3d, [batch_size, num_point * up_ratio, -1])
with tf.variable_scope('normal_predict'):
dense_normal_offset = tf_util.conv2d(concat,
up_ratio * 3, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
activation_fn=None,
bn=None,
is_training=is_training,
scope='normal_offset_conv1',
weight_decay=weight_decay)
dense_normal_offset = tf.reshape(
dense_normal_offset, [batch_size, num_point, up_ratio, 3])
sparse_normal = tf.convert_to_tensor([0, 0, 1], dtype=tf.float32)
sparse_normal = tf.expand_dims(sparse_normal, axis=0)
sparse_normal = tf.expand_dims(sparse_normal, axis=0)
sparse_normal = tf.expand_dims(sparse_normal, axis=0)
sparse_normal = tf.tile(sparse_normal,
[batch_size, num_point, 1, 1])
sparse_normal = tf.matmul(sparse_normal, affine_T)
sparse_normal = tf.nn.l2_normalize(sparse_normal, axis=-1)
dense_normal = tf.tile(sparse_normal,
[1, 1, up_ratio, 1]) + dense_normal_offset
dense_normal = tf.nn.l2_normalize(dense_normal, axis=-1)
dense_normal = tf.transpose(dense_normal, perm=[0, 2, 1, 3])
dense_normal = tf.reshape(dense_normal,
[batch_size, num_point * up_ratio, -1])
with tf.variable_scope('up_layer'):
if not np.isscalar(bradius):
bradius_expand = tf.expand_dims(tf.expand_dims(bradius,
axis=-1),
axis=-1)
else:
bradius_expand = bradius
bradius_expand = bradius_expand
grid = tf.expand_dims(uv_3d * bradius_expand, axis=2)
concat_up = tf.tile(concat, (1, up_ratio, 1, 1))
concat_up = tf.concat([concat_up, grid], axis=-1)
concat_up = tf_util.conv2d(concat_up,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='up_layer1',
bn_decay=bn_decay,
weight_decay=weight_decay)
concat_up = tf_util.dropout(concat_up,
keep_prob=0.6,
is_training=is_training,
scope='up_dg1')
concat_up = tf_util.conv2d(concat_up,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='up_layer2',
bn_decay=bn_decay,
weight_decay=weight_decay)
concat_up = tf_util.dropout(concat_up,
keep_prob=0.6,
is_training=is_training,
scope='up_dg2')
# get xyz
coord_z = tf_util.conv2d(concat_up,
1, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
is_training=is_training,
scope='fc_layer',
weight_decay=weight_decay)
coord_z = tf.reshape(coord_z, [batch_size, up_ratio, num_point, 1])
coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3])
coord_z = tf.concat(
[tf.zeros_like(coord_z),
tf.zeros_like(coord_z), coord_z], axis=-1)
coord_z = tf.matmul(coord_z, affine_T)
coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3])
coord_z = tf.reshape(coord_z, [batch_size, num_point * up_ratio, -1])
coord = uv_3d + coord_z
return coord, dense_normal, tf.squeeze(sparse_normal, [2])
def get_model_other(point_cloud, is_training, bn_decay=None):
"""
B: batch size;
N: number of points,
C: channels;
k: number of nearest neighbors
point_cloud: B*N*C
"""
end_points = {}
minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1))
batch_size = point_cloud.get_shape()[0] # .value
# # 1. graph for first EdgeConv B N C=6
adj_matrix = tf_util.pairwise_distance(
point_cloud[:, :, :para.dim]) # B N C=6 => B*N*N
# adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:para.dim]) # B N C=6 => B*N*N
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn1'] = allSF_dist
point_cloud = tf.expand_dims(point_cloud[:, :, :para.dim], axis=-2)
# point_cloud = tf.expand_dims(point_cloud[:, :, 1:para.dim], axis=-2)
edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn1',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net1 = net
# # 2. graph for second EdgeConv B N C=64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn2'] = allSF_dist
# net: B*N*1*6+64=71
net = tf.concat([point_cloud, net1], axis=-1)
# edge_feature: B*N*k*142
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn2',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net2 = net
# 3. graph for third EdgeConv B N C=64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn3'] = allSF_dist
# net: B*N*1*6+64+64=134
net = tf.concat([point_cloud, net1, net2], axis=-1)
# edge_feature: B*N*k*268
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn3',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net3 = net
# 4. graph for fourth EdgeConv B N C=64
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=20)
# get the distance to minSF of 1024 points
allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
end_points['knn4'] = allSF_dist
# net: B*N*1*6+64+64+64=198
net = tf.concat([point_cloud, net1, net2, net3], axis=-1)
# edge_feature: B*N*k*396
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
net = tf_util.conv2d(edge_feature,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='dgcnn4',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keepdims=True)
net4 = net
# input: B*N*1*6+64+64+128+128 = 326 => net: B*N*1*1024
net = tf_util.conv2d(tf.concat([point_cloud, net1, net2, net3, net4],
axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='agg',
bn_decay=bn_decay)
# net: B*1*1*1024
# SF_features = tf.gather(net, indices=minSF, axis=1, batch_dims=1)
net = tf.reduce_max(net, axis=1, keepdims=True)
# SF_all = tf.concat([SF_features,net], axis=-1)
# net: B*1024
net = tf.squeeze(net)
# net: B*2048
# net = tf.squeeze(SF_all)
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
print(net.get_shape())
end_points['global_feature'] = net
# Fully connected end_points: classifier
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['fc1'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
end_points['fc2'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net,
para.outputClassN,
activation_fn=None,
scope='fc3')
end_points['fc3'] = net
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
with tf.variable_scope('transform_net') as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
# KNN local search
#knn_point = KNN_search(point_cloud_transformed, KNN=KNN, name_scope='KNN_search') # 32 x 1024 x KNN x 6
#knn_point = tf.expand_dims(knn_point, axis=-1) # 32 x 1024 x KNN x 3 x 1
with tf.variable_scope('group_sampling') as sc:
KNN = 16
point_cloud_transformed, _, _, nn_points = sample_and_group(
npoint=1024,
radius=0.1,
nsample=KNN,
xyz=point_cloud_transformed,
points=None,
knn=True,
use_xyz=False)
point_cloud_transformed = tf.expand_dims(point_cloud_transformed, axis=-1)
net1 = tf_util.conv2d(point_cloud_transformed,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1_1',
bn_decay=bn_decay) # 32 x 1024 x 1 x 64
net1 = tf.tile(net1, multiples=[1, 1, KNN, 1]) # 32 x 1024 x 16 x 64
net1 = tf.expand_dims(net1, axis=-2)
nn_points = tf.expand_dims(nn_points, axis=-1) # 32 x 1024 x 16 x 3 x 1
net = tf_util.conv3d(nn_points,
64, [1, 1, 3],
padding='VALID',
stride=[1, 1, 1],
bn=True,
is_training=is_training,
scope='conv1_2',
bn_decay=bn_decay) # 32 x 1024 x 16 x 1 x 64
concat = tf.concat(values=[net, net1], axis=-1)
net = tf_util.conv3d(concat,
128, [1, 1, 1],
padding='VALID',
stride=[1, 1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv3d(net,
128, [1, 1, 1],
padding='VALID',
stride=[1, 1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
# local pooling: merge local feature
pool_k = tf_util.max_pool3d(net,
kernel_size=[1, KNN, 1],
stride=[1, 2, 2],
padding='VALID',
scope='pool_k') # 32 x 1024 x 1 x 1 x 128
net1 = tf.squeeze(pool_k, axis=2)
# VLAD layer
net2, index = VLAD(net1,
16,
is_training,
bn_decay,
layer_name='VLAD_layer1')
net = VLAD_layer(net2,
net1,
16,
is_training,
bn_decay,
layer_name='VLAD_layer2')
# classification
with tf.name_scope('fc_layer'):
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, index
