def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Set abstraction layers
# Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4).
# So we only use NCHW for layer 1 until this issue can be resolved.
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True)
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image, 64, [1,3],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = tf_util.conv2d(net_transformed, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1])
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_instance_seg_v2_net(point_cloud, one_hot_vec,
is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points,
128, [0.2,0.4,0.8], [32,64,128],
[[32,32,64], [64,64,128], [64,96,128]],
is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points,
32, [0.4,0.8,1.6], [64,64,128],
[[64,64,128], [128,128,256], [128,128,256]],
is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points,
npoint=None, radius=None, nsample=None, mlp=[128,256,1024],
mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2)
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points,
[128,128], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points,
[128,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz,
tf.concat([l0_xyz,l0_points],axis=-1), l1_points,
[128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True,
is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.7,
is_training=is_training, scope='dp1')
logits = tf_util.conv1d(net, 2, 1,
padding='VALID', activation_fn=None, scope='conv1d-fc2')
return logits, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [16,32,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1', use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2,0.4,0.8], [32,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
input_image = tf.expand_dims(point_cloud, -1)
# Point functions (MLP implemented as conv2d)
net = tf_util.conv2d(input_image, 64, [1,3],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# npoints = [256, 64, 10]
# npoints = [512, 128, 10]
# npoints = [512, 128, 8]
npoints = [512, 128, 32]
ANCHOR = 27
l1_xyz, l1_points, l1_nn, l1_offset = sample_and_group(npoint=npoints[0],
radius=0.2,
nsample=32,
xyz=l0_xyz,
points=None,
knn=False,
use_xyz=True)
l2_xyz, l2_points, l2_nn, l2_offset = sample_and_group(npoint=npoints[1],
radius=0.4,
nsample=64,
xyz=l1_xyz,
points=None,
knn=False,
use_xyz=True)
l3_xyz, l3_points, l3_nn, l3_offset = sample_and_group(npoint=npoints[2],
radius=0.8,
nsample=64,
xyz=l2_xyz,
points=None,
knn=False,
use_xyz=True)
# # (1024, 3) -> (1024, 64) fc-0
# # (1024, 64) -> (1024, 64) conv-1
# # (1024, 64) -> (1024, 128) fc-1
# # (1024, 128) -> (1024, 256) conv-2
# # (1024, 256) -> (1024, 256) fc-2
# k_list = [10, 10, 10]
# cube_list = [0.2, 0.2, 0.2]
# channels = [64, 128, 256]
# xyz_list = [l0_xyz, l0_xyz, l0_xyz, l0_xyz]
# # (1024, 3) -> (1024, 64) fc-0
# # (1024, 64) -> (512, 64) conv-1
# # (512, 64) -> (512, 256) fc-1
# # (512, 256) -> (10, 256) conv-2
# # (10, 1024) -> (10, 1024) fc-2
# k_list = [32, 64, 64]
# cube_list = [0.1, 0.2, 0.4]
# channels = [64, 256, 512]
# xyz_list = [l0_xyz, l1_xyz, l2_xyz, l3_xyz]
# # (1024, 3) -> (1024, 64) fc-0
# # (1024, 64) -> (512, 64) conv-1
# # (512, 64) -> (512, 256) fc-1
# # (512, 256) -> (10, 256) conv-2
# # (10, 1024) -> (10, 1024) fc-2
# k_list = [32, 64, 128]
# cube_list = [0.2, 0.4, 1.0]
# channels = [64, 256, 512]
# xyz_list = [l0_xyz, l1_xyz, l2_xyz, l3_xyz]
# # (1024, 3) -> (1024, 64) fc-0
# # (1024, 64) -> (512, 64) conv-1
# # (512, 64) -> (512, 256) fc-1
# # (512, 256) -> (10, 256) conv-2
# # (10, 1024) -> (10, 1024) fc-2
# k_list = [10, 10, 10]
# cube_list = [0.1, 0.2, 0.4]
# channels = [64, 256, 1024]
# xyz_list = [l0_xyz, l1_xyz, l2_xyz, l3_xyz]
# k_list = [10, 10, 10, 10, 10]
# cube_list = [0.2, 0.2, 0.4, 0.4, 0.8]
# channels = [64, 64, 256, 256, 1024]
# xyz_list = [l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz]
# # (1024, 3) -> (1024, 32) fc-0
# # (1024, 32) -> (1024, 32) conv-1-1 K=32 l0 -> l0
# # (1024, 32) -> (1024, 32) fc-1-1
# # (1024, 32) -> (512, 32) conv-1-2 K=32 l0 -> l1
# # (512, 32) -> (512, 128) fc-1-2
# # (512, 128) -> (512, 128) conv-2-1 K=16 l1 -> l1
# # (512, 128) -> (512, 128) fc-2-1
# # (512, 128) -> (128, 128) conv-2-2 K=16 l1 -> l2
# # (128, 128) -> (128, 512) fc-2-2
# # (128, 512) -> (128, 512) conv-3-1 K=8 l2 -> l2
# # (128, 512) -> (128, 512) fc-3-1
# # (128, 512) -> (8, 512) conv-3-2 K=8 l2 -> l3
# # (8, 512) -> (8, 512) fc 3-2
# k_list = [32, 32, 16, 16, 8, 8]
# cube_list = [0.2, 0.2, 0.4, 0.4, 0.8, 0.8]
# channels = [32, 32, 128, 128, 512, 512]
# xyz_list = [l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz]
# (1024, 3) -> (1024, 32) fc-0
# (1024, 32) -> (1024, 32) conv-1-1 K=32 l0 -> l0
# (1024, 32) -> (1024, 32) fc-1-1
# (1024, 32) -> (512, 32) conv-1-2 K=32 l0 -> l1
# (512, 32) -> (512, 128) fc-1-2
# (512, 128) -> (512, 128) conv-2-1 K=16 l1 -> l1
# (512, 128) -> (512, 128) fc-2-1
# (512, 128) -> (128, 128) conv-2-2 K=16 l1 -> l2
# (128, 128) -> (128, 512) fc-2-2
# (128, 512) -> (128, 512) conv-3-1 K=16 l2 -> l2
# (128, 512) -> (128, 512) fc-3-1
# (128, 512) -> (1, 512) conv-3-2 K=128 l2 -> l3
# (1, 512) -> (1, 512) fc 3-2
# k_list = [32, 32, 16, 16, 16, 128]
# cube_list = [0.2, 0.2, 0.4, 0.4, 0.8, 1.2]
# channels = [32, 32, 128, 128, 512, 512]
# xyz_list = [l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz]
k_list = [32, 32, 16, 16, 16, 16]
cube_list = [0.2, 0.2, 0.4, 0.4, 0.8, 0.8]
channels = [32, 32, 128, 128, 512, 512]
xyz_list = [l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz]
# # (1024, 3) -> (1024, 32) fc-0
# # (1024, 32) -> (1024, 32) conv-1-1 K=10 l0 -> l0
# # (1024, 32) -> (1024, 32) fc-1-1
# # (1024, 32) -> (1024, 32) conv-1-2 K=10 l0 -> l0
# # (1024, 32) -> (1024, 32) fc-1-2
# # (1024, 32) -> (1024, 32) conv-1-3 K=10 l0 -> l0
# # (1024, 32) -> (1024, 32) fc-1-3
# # (1024, 32) -> (512, 32) conv-1-4 K=10 l0 -> l1
# # (512, 32) -> (512, 128) fc-1-4
# # (512, 128) -> (512, 128) conv-2-1 K=10 l1 -> l1
# # (512, 128) -> (512, 128) fc-2-1
# # (512, 128) -> (128, 128) conv-2-2 K=10 l1 -> l2
# # (128, 128) -> (128, 512) fc-2-2
# # (128, 512) -> (128, 512) conv-3-1 K=10 l2 -> l2
# # (128, 512) -> (128, 512) fc-3-1
# # (128, 512) -> (10, 512) conv-3-2 K=10 l2 -> l3
# # (10, 512) -> (10, 512) fc 3-2
# k_list = [10, 10, 10, 10, 10, 10, 10, 10]
# cube_list = [0.2, 0.2, 0.2, 0.2, 0.4, 0.4, 0.8, 0.8]
# channels = [32, 32, 32, 32, 128, 128, 512, 512]
# xyz_list = [l0_xyz, l0_xyz, l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz]
nlayers = len(xyz_list) - 1
if True:
offset_list = [
tf.py_func(get_knn, [_xyz, _xyz2, _k], [tf.float32, tf.int64])
for _xyz, _xyz2, _k in zip(xyz_list[:-1], xyz_list[1:], k_list)
]
nn_list = [_offset[1] for _offset in offset_list]
offset_list = [_offset[0] for _offset in offset_list]
offset00, _ = tf.py_func(get_knn,
[xyz_list[0], xyz_list[0], k_list[0]],
[tf.float32, tf.int64])
else:
offset_list = [l1_offset, l2_offset, l3_offset]
nn_list = [l1_nn, l2_nn, l3_nn]
nn_list = [tf.cast(nn, tf.int64) for nn in nn_list]
alpha_list = [
cont_filter_interp(_offset, _cube, ANCHOR)
for (_offset, _cube) in zip(offset_list, cube_list)
]
def _fc(points, w, bn=True):
N = points.shape[0]
in_channels = w.shape[0]
out_channels = w.shape[1]
points_ = tf.reshape(points, [-1, in_channels])
points_ = tf.matmul(points_, w)
points = tf.reshape(points_, [N, -1, out_channels])
if bn:
points = tf.contrib.layers.batch_norm(points,
center=True,
scale=True,
is_training=is_training,
decay=bn_decay,
updates_collections=None)
return points
def _conv_block(points, knn, alpha, c_in, c_out):
conv_init = tf.initializers.uniform_unit_scaling(1.0)
conv_w = tf.get_variable('convw', [
ANCHOR,
c_in,
],
initializer=conv_init,
trainable=True)
fc_w = tf.get_variable('fcw', [c_in, c_out],
initializer=tf.initializers.truncated_normal(
0.0, 2.0 / np.sqrt(float(c_out))),
trainable=True)
points = cont_filter_conv(points, knn, alpha, conv_w)
points = _fc(points, fc_w, bn=True)
points = tf.nn.relu(points)
return points
c_in = channels[0]
c_init = 3
# points = tf.ones_like(l0_xyz[:, :, :1])
# points = tf.reduce_mean(offset00, [2]) # [B, M, K, D]
# points = tf.reshape(points, [16, 1024, 3])
points = l0_xyz
fc0 = tf.get_variable('w0', [c_init, c_in],
initializer=tf.initializers.truncated_normal(
0.0, 2.0 / np.sqrt(float(c_in))))
points = _fc(points, fc0, bn=False)
print(points)
for layer in range(nlayers):
with tf.variable_scope('l{}'.format(layer)):
print(layer, len(alpha_list))
alpha = alpha_list[layer]
nn = nn_list[layer]
c_out = channels[layer]
points = _conv_block(points, nn, alpha, c_in, c_out)
c_in = c_out
# Fully connected layers
l3_points = tf.reduce_max(points, [1]) # [B, C]
# l3_points = tf.reduce_mean(points, [1]) # [B, C]
net = tf.reshape(l3_points, [batch_size, channels[-1]])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \
batch_size, num_point, weight_decay, graphnum, featnum, bn_decay=None):
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -1)
k = 30
adj = tf_util.pairwise_distance(point_cloud)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k)
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3,
is_dist=True)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -2)
adj = tf_util.pairwise_distance(point_cloud_transformed)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k)
out1 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv1',
bn_decay=bn_decay,
is_dist=True)
out2 = tf_util.conv2d(out1,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv2',
bn_decay=bn_decay,
is_dist=True)
net_max_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)
net_mean_1 = tf.reduce_mean(out2, axis=-2, keep_dims=True)
out3 = tf_util.conv2d(tf.concat([net_max_1, net_mean_1], axis=-1),
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv3',
bn_decay=bn_decay,
is_dist=True)
out3A, net_max_1A, net_mean_1A = tf_util.offset_deform(
input_image,
out3,
scope="trans_conv0",
num_neighbor=k,
num_graph=graphnum[0],
num_feat=featnum[0],
weight_decay=weight_decay,
is_training=is_training,
bn_decay=bn_decay)
out5, net_max_2, net_mean_2 = tf_util.offset_deform(
input_image,
out3A,
scope="trans_conv1",
num_neighbor=k,
num_graph=graphnum[0],
num_feat=featnum[0],
weight_decay=weight_decay,
is_training=is_training,
bn_decay=bn_decay)
out7, net_max_3, net_mean_3 = tf_util.offset_deform(
input_image,
out5,
scope="trans_conv2",
num_neighbor=k,
num_graph=graphnum[1],
num_feat=featnum[1],
weight_decay=weight_decay,
is_training=is_training,
bn_decay=bn_decay)
'''adj = tf_util.pairwise_distance(tf.squeeze(trans2, axis=-2))
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(tf.concat([out5,trans2], axis = -1), nn_idx=nn_idx, k=k)
out6 = tf_util.conv2d(edge_feature, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training, weight_decay=weight_decay,
scope='adj_conv6', bn_decay=bn_decay, is_dist=True)
net_max_3 = tf.reduce_max(out6, axis=-2, keep_dims=True)
net_mean_3 = tf.reduce_mean(out6, axis=-2, keep_dims=True)
out7 = tf_util.conv2d(tf.concat([net_max_3, net_mean_3], axis=-1), 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training, weight_decay=weight_decay,
scope='adj_conv7', bn_decay=bn_decay, is_dist=True)'''
out8 = tf_util.conv2d(tf.concat([out3, out5, out7], axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='adj_conv13',
bn_decay=bn_decay,
is_dist=True)
out_max = tf_util.max_pool2d(out8, [num_point, 1],
padding='VALID',
scope='maxpool')
one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num])
one_hot_label_expand = tf_util.conv2d(one_hot_label_expand,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='one_hot_label_expand',
bn_decay=bn_decay,
is_dist=True)
out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand])
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat = tf.concat(axis=3,
values=[
expand, net_max_1, net_mean_1, out3, net_max_2,
net_mean_2, out5, net_max_3, net_mean_3, out7, out8
])
net2 = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv1',
weight_decay=weight_decay,
is_dist=True)
net2 = tf_util.dropout(net2,
keep_prob=0.6,
is_training=is_training,
scope='seg/dp1')
net2 = tf_util.conv2d(net2,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv2',
weight_decay=weight_decay,
is_dist=True)
net2 = tf_util.dropout(net2,
keep_prob=0.6,
is_training=is_training,
scope='seg/dp2')
net2 = tf_util.conv2d(net2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv3',
weight_decay=weight_decay,
is_dist=True)
net2 = tf_util.conv2d(net2,
part_num, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
scope='seg/conv4',
weight_decay=weight_decay,
is_dist=True)
net2 = tf.reshape(net2, [batch_size, num_point, part_num])
return net2
def get_scene_model(point_cloud,
is_training,
num_class,
sigma,
bn_decay=None,
weight_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud[:, :, :3]
l0_points = point_cloud
l0_points_xyz = point_cloud[:, :, :3]
l0_points_rgb = point_cloud[:, :, 3:]
# Feature encoding layers
l1_xyz, l1_points_xyz = feature_encoding_layer(l0_xyz,
l0_points_xyz,
npoint=2048,
radius=0.1,
sigma=sigma,
K=8,
mlp=[32, 32, 32],
akc_channel=3,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer1_xyz')
_, l1_points_rgb = feature_encoding_layer_extra(l0_xyz,
l1_xyz,
l0_points_rgb,
npoint=2048,
radius=0.1,
sigma=sigma,
K=8,
mlp=[32, 32, 32],
akc_channel=3,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer1_rgb')
l1_points = tf.concat([l1_points_xyz, l1_points_rgb], -1)
l1_xyz_1024, l1_points_1024 = feature_encoding_layer(
l1_xyz,
l1_points,
npoint=1024,
radius=0.1,
sigma=sigma,
K=8,
mlp=[32, 32, 64],
akc_channel=16,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer1_1024')
l1_xyz_512, l1_points_512 = feature_encoding_layer(
l1_xyz_1024,
l1_points_1024,
npoint=512,
radius=0.1,
sigma=sigma,
K=8,
mlp=[64, 64, 64],
akc_channel=32,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer1_512')
l2_xyz, l2_points = feature_encoding_layer(l1_xyz_512,
l1_points_512,
npoint=256,
radius=0.2,
sigma=2 * sigma,
K=8,
mlp=[64, 64, 128],
akc_channel=32,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer2')
l2_xyz_128, l2_points_128 = feature_encoding_layer(
l2_xyz,
l2_points,
npoint=128,
radius=0.2,
sigma=2 * sigma,
K=8,
mlp=[128, 128, 128],
akc_channel=None,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer2_128')
l3_xyz, l3_points = feature_encoding_layer(l2_xyz_128,
l2_points_128,
npoint=64,
radius=0.4,
sigma=4 * sigma,
K=8,
mlp=[128, 128, 256],
akc_channel=None,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer3')
l4_xyz, l4_points = feature_encoding_layer(l3_xyz,
l3_points,
npoint=36,
radius=0.8,
sigma=8 * sigma,
K=8,
mlp=[256, 256, 512],
akc_channel=None,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='layer4')
external_l5_xyz, external_l5_points = feature_encoding_layer(
l4_xyz,
l4_points,
npoint=8,
radius=1.6,
sigma=8 * sigma,
K=8,
mlp=[512, 512, 512],
akc_channel=None,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
scope='external_layer5')
external_l6_scene_feature = tf.reduce_mean(external_l5_points,
axis=1,
keepdims=True)
external_scene_feature = tf_util.dropout(external_l6_scene_feature,
keep_prob=0.5,
is_training=is_training,
scope='external_dp')
external_scene_feature = tf_util.conv1d(external_scene_feature,
num_class,
1,
padding='VALID',
activation_fn=None,
weight_decay=weight_decay,
scope='external_fc')
# Feature decoding layers
l3_points = feature_decoding_layer(l3_xyz,
l4_xyz,
l3_points,
l4_points,
0.8,
8 * sigma,
8, [512, 512],
is_training,
bn_decay,
weight_decay,
scope='fa_layer1')
l2_points_128 = feature_decoding_layer(l2_xyz_128,
l3_xyz,
l2_points_128,
l3_points,
0.4,
4 * sigma,
8, [256, 256],
is_training,
bn_decay,
weight_decay,
scope='fa_layer2_128')
l2_points = feature_decoding_layer(l2_xyz,
l2_xyz_128,
l2_points,
l2_points_128,
0.4,
4 * sigma,
8, [256, 256],
is_training,
bn_decay,
weight_decay,
scope='fa_layer2')
l1_points_512 = feature_decoding_layer(l1_xyz_512,
l2_xyz,
l1_points_512,
l2_points,
0.2,
2 * sigma,
8, [256, 256],
is_training,
bn_decay,
weight_decay,
scope='fa_layer3_512')
l1_points_1024 = feature_decoding_layer(l1_xyz_1024,
l1_xyz_512,
l1_points_1024,
l1_points_512,
0.2,
2 * sigma,
8, [256, 256],
is_training,
bn_decay,
weight_decay,
scope='fa_layer3_1024')
l1_points = feature_decoding_layer(l1_xyz,
l1_xyz_1024,
l1_points,
l1_points_1024,
0.2,
2 * sigma,
8, [256, 128],
is_training,
bn_decay,
weight_decay,
scope='fa_layer3')
l0_points = feature_decoding_layer(l0_xyz,
l1_xyz,
l0_points,
l1_points,
0.1,
sigma,
8, [128, 128, 128],
is_training,
bn_decay,
weight_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay,
weight_decay=weight_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
weight_decay=weight_decay,
scope='fc2')
return net, end_points, external_scene_feature
def get_instance_seg_v1_net(point_cloud, one_hot_vec,
is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v1 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
net = tf.expand_dims(point_cloud, 2)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
point_feat = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(point_feat, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
global_feat = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3)
global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])
concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand])
net = tf_util.conv2d(concat_feat, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv6', bn_decay=bn_decay)
net = tf_util.conv2d(net, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv7', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv8', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv9', bn_decay=bn_decay)
net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5)
logits = tf_util.conv2d(net, 2, [1,1],
padding='VALID', stride=[1,1], activation_fn=None,
scope='conv10')
logits = tf.squeeze(logits, [2]) # BxNxC
return logits, end_points
def get_model(point_cloud, is_training, POINT_DIM, bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -1)
# CONV
net = tf_util.conv2d(input_image,
64, [1, POINT_DIM],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
points_feat1 = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# MAX
pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point, 1],
padding='VALID',
scope='maxpool1')
# FC
pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1])
pc_feat1 = tf_util.fully_connected(pc_feat1,
256,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
pc_feat1 = tf_util.fully_connected(pc_feat1,
128,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
print(pc_feat1)
# CONCAT
pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]),
[1, num_point, 1, 1])
points_feat1_concat = tf.concat(axis=3,
values=[points_feat1, pc_feat1_expand])
# CONV
net = tf_util.conv2d(points_feat1_concat,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv6')
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv7')
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.conv2d(net,
13, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
scope='conv8')
net = tf.squeeze(net, [2])
return net
def get_instance_seg_v2_net(point_cloud, one_hot_vec, is_training, bn_decay,
end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 1])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
32, [0.4, 0.8, 1.6], [64, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[128, 256, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature Propagation layers
l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2)
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat([l0_xyz, l0_points], axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='conv1d-fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
logits = tf_util.conv1d(net,
2,
1,
padding='VALID',
activation_fn=None,
scope='conv1d-fc2')
return logits, end_points
def get_model_rbf0_gan(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:
transform = input_transform_net_no_bn(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3)
#centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32)
centroids = tf.get_variable('centroids', [1, 1, 3, 1024],
initializer=tf.constant_initializer(
0.2 * np.random.randn(1, 1, 3, 1024)),
dtype=tf.float32)
feature = tf.tile(point_cloud_transformed, [1, 1, 1, 1024])
bias = tf.tile(centroids, [batch_size, 1024, 1, 1])
net = tf.subtract(feature, bias)
net = tf.norm(net, axis=2, keep_dims=True)
net = tf.exp(-net)
# Symmetric function: max pooling
features = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.reshape(features, [batch_size, -1])
#net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training,
# scope='fc0', bn_decay=bn_decay)
#net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
# scope='dp1')
net = tf_util.fully_connected(net,
512,
bn=False,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=False,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 41, activation_fn=None, scope='fc3')
return net, end_points, features, centroids
def get_model(sequence,
is_training,
num_classes=10,
bn_decay=0.999,
weight_decay=0.0001,
sn=4,
pool_t=False,
pool_first=False,
freeze_bn=False):
""" SeNot Net, input is BxTxHxWx3, output Bx10 """
bsize = sequence.get_shape()[0].value
end_points = {}
channel_stride = [(64, 1), (128, 2), (256, 2), (512, 2)]
# res block options
num_blocks = [2, 2, 2, 2]
topks = [None, sn, sn, None]
shrink_ratios = [None, 2, None, None]
net = tf_util.conv3d(sequence,
64, [1, 3, 3],
stride=[1, 1, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training,
weight_decay=weight_decay,
freeze_bn=freeze_bn,
scope='conv0')
net = tf_util.max_pool3d(net, [1, 3, 3],
stride=[1, 2, 2],
scope='pool0',
padding='SAME')
for gp, cs in enumerate(channel_stride):
n_channels = cs[0]
stride = cs[1]
with tf.variable_scope('group{}'.format(gp)):
for i in range(num_blocks[gp]):
with tf.variable_scope('block{}'.format(i)):
end_points['res{}_{}_in'.format(gp, i)] = net
if i == 0:
net_bra = tf_util.conv3d(net, n_channels, [1, 3, 3], stride=[1, stride, stride], bn=True, bn_decay=bn_decay, \
is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1')
else:
net_bra = tf_util.preact_bn_for_conv3d(net, is_training=is_training, bn_decay=bn_decay, \
scope='preact', freeze_bn=freeze_bn)
net_bra = tf_util.conv3d(net_bra, n_channels, [1, 3, 3], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \
is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1')
net_bra = tf_util.conv3d(net_bra, n_channels, [1, 3, 3], stride=[1, 1, 1], bn=False, bn_decay=bn_decay, \
is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv2')
if net.get_shape()[-1].value != n_channels:
net = tf_util.conv3d(net, n_channels, [1, 1, 1], stride=[1, stride, stride], bn=False, bn_decay=bn_decay, \
is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='convshortcut')
net = net + net_bra
if i == 1:
net = tf_util.preact_bn_for_conv3d(net, is_training=is_training, bn_decay=bn_decay, \
scope='bnlast', freeze_bn=freeze_bn)
end_points['res{}_{}_mid'.format(gp, i)] = net
if topks[gp] is not None:
c = net.get_shape()[-1].value
net_pointnet, end_point = net_utils.senota_module(net, k=topks[gp], mlp=[c//4,c//2], scope='pointnet', is_training=is_training, bn_decay=bn_decay, \
weight_decay=weight_decay, distance='l2', activation_fn=None, freeze_bn=freeze_bn, shrink_ratio=shrink_ratios[gp])
net += net_pointnet
end_points['pointnet{}_{}'.format(gp, i)] = end_point
end_points['after_pointnet{}_{}'.format(gp, i)] = net
net = tf.nn.relu(net)
end_points['res{}_{}_out'.format(gp, i)] = net
net = tf.reduce_mean(net, [1, 2, 3])
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp')
net = tf_util.fully_connected(net,
num_classes,
activation_fn=None,
weight_decay=weight_decay,
scope='fc')
return net, end_points
def get_model_elm(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
random_weights = tf.constant(np.random.randn(3, 4096), dtype=tf.float32)
random_weights1 = tf.expand_dims(random_weights, 0)
random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #2
random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #4
random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #8
random_weights1 = tf.concat([random_weights1, random_weights1],
axis=0) #16
random_weights1 = tf.concat([random_weights1, random_weights1],
axis=0) #32
net = tf.matmul(point_cloud, random_weights1)
net = tf.expand_dims(net, 2)
# Symmetric function: max pooling
features = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.reshape(features, [batch_size, -1])
net = tf_util.fully_connected(net,
1024,
bn=True,
is_training=is_training,
scope='fc0',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points, features
def get_model_rbf_transform(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
c1 = 64
#centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32)
centroids = tf.get_variable('centroids', [1, 1, 3, c1],
initializer=tf.constant_initializer(
0.5 * np.random.randn(1, 1, 3, c1)),
dtype=tf.float32)
net = tf.subtract(
tf.tile(tf.expand_dims(point_cloud_transformed, 3), [1, 1, 1, c1]),
tf.tile(centroids, [batch_size, num_point, 1, 1]))
net = tf.norm(net, axis=2, keep_dims=True)
net = tf.exp(-net)
with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
c2 = 256
#centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32)
centroids2 = tf.get_variable('centroids2', [1, 1, c2, 64],
initializer=tf.constant_initializer(
0.5 * np.random.randn(1, 1, c2, 64)),
dtype=tf.float32)
net = tf.subtract(tf.tile(net_transformed, [1, 1, c2, 1]),
tf.tile(centroids2, [batch_size, num_point, 1, 1]))
net = tf.norm(net, axis=3, keep_dims=True)
net = tf.exp(-net)
# Symmetric function: max pooling
features = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.reshape(features, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points, features, centroids
def get_model_rbf3(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3)
#centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32)
c1 = 512
c2 = 8
centroids = tf.get_variable('centroids', [1, 1, 3, c1],
initializer=tf.constant_initializer(
np.random.randn(1, 1, 3, c1)),
dtype=tf.float32)
sub_centroids = tf.get_variable('sub_centroids', [1, 1, 3, c2],
initializer=tf.constant_initializer(
0.05 * np.random.randn(1, 1, 3, c2)),
dtype=tf.float32)
#sub_centroids = tf.constant(0.05*np.random.randn(1, 1, 3, c2), dtype=tf.float32)
sub_bias = tf.add(
tf.tile(tf.expand_dims(sub_centroids, 4), [1, 1, 1, 1, c1]),
tf.tile(tf.expand_dims(centroids, 3), [1, 1, 1, c2, 1]))
sub_bias = tf.tile(sub_bias, [batch_size, 1024, 1, 1, 1])
sub_feature = tf.tile(tf.expand_dims(point_cloud_transformed, 4),
[1, 1, 1, c2, c1])
sub_net = tf.exp(-tf.square(
tf.norm(
tf.subtract(sub_feature, sub_bias), ord=3, axis=2, keep_dims=True))
)
sub_net = tf.squeeze(sub_net)
sub_net = tf.transpose(sub_net, perm=[0, 1, 3, 2])
sub_net = tf_util.max_pool2d(sub_net, [num_point, 1],
stride=[1, 1],
padding='VALID',
scope='maxpool')
#sub_net = tf_util.conv2d(sub_net, 16, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='mini_conv1', bn_decay=bn_decay)
sub_net = tf_util.conv2d(sub_net,
2, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='mini_conv2',
bn_decay=bn_decay)
sub_net = tf.squeeze(sub_net)
feature = tf.tile(point_cloud_transformed, [1, 1, 1, c1])
bias = tf.tile(centroids, [batch_size, 1024, 1, 1])
net = tf.subtract(feature, bias)
#net = tf.exp(net)
net = tf.norm(net, ord=3, axis=2, keep_dims=True)
net = tf.exp(-tf.square(net))
net = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.expand_dims(tf.squeeze(net), 2)
features = tf.concat([net, sub_net], axis=2)
# Symmetric function: max pooling
#features = tf_util.max_pool2d(net, [num_point,1],
# padding='VALID', scope='maxpool')
net = tf.reshape(features, [batch_size, -1])
#net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training,
# scope='fc0', bn_decay=bn_decay)
#net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
# scope='dp1')
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points, features, centroids
def get_model_rbf(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3)
c1 = 1024
#centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32)
centroids = tf.get_variable('centroids', [1, 1, 3, c1],
initializer=tf.constant_initializer(
0.5 * np.random.randn(1, 1, 3, c1)),
dtype=tf.float32)
#the per-centroids weights to change the shape of the multi-norm
weights = tf.get_variable(
'weights',
[1, 1, 4, c1],
initializer=tf.constant_initializer(0.01 *
np.random.randn(1, 1, 3, c1)),
)
feature = tf.tile(point_cloud_transformed, [1, 1, 1, c1])
bias = tf.tile(centroids, [batch_size, num_point, 1, 1])
net = tf.subtract(feature, bias)
net = tf.exp(net)
net = tf.exp(-tf.concat(
[
tf.norm(net, ord=0.5, axis=2, keep_dims=True),
#tf.norm(net, ord=0.8, axis=2, keep_dims=True),
tf.norm(net, ord=1, axis=2, keep_dims=True),
#tf.norm(net, ord=1.5, axis=2, keep_dims=True),
tf.norm(net, ord=2, axis=2, keep_dims=True),
#tf.norm(net, ord=3, axis=2, keep_dims=True),
#tf.norm(net, ord=4, axis=2, keep_dims=True),
tf.norm(net, ord=np.inf, axis=2, keep_dims=True),
],
axis=2))
net = tf.multiply(net, tf.tile(weights, [batch_size, num_point, 1, 1]))
#net = tf.exp(-net)
# Symmetric function: max pooling
features = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.transpose(features, perm=[0, 1, 3, 2])
net = tf_util.conv2d(net,
3, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='mini_conv1',
bn_decay=bn_decay)
net = tf.reshape(net, [batch_size, -1])
#net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training,
# scope='fc0', bn_decay=bn_decay)
#net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
# scope='dp1')
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points, features, centroids
def get_model(point_cloud, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """
############# PointNet++ 分割部分，先采样，再分组，并分层提取出点云特征######################
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
# Set abstraction layers, 分层特征提取（加SSG策略，single scale grouping）
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=512,
radius=0.2,
nsample=64,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=128,
radius=0.4,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature Propagation layers, 对特征进行插值，恢复到之前的点数，最后对每个点进行分类
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat([l0_xyz, l0_points], axis=-1),
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2') ##(B,L,50)
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
k = 20
print(point_cloud)
adj_matrix = tf_util.pairwise_distance(point_cloud)
print(adj_matrix)
nn_idx = tf_util.knn(adj_matrix, k=k)
print(nn_idx)
edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k)
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
adj_matrix = tf_util.pairwise_distance(point_cloud_transformed)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(point_cloud_transformed, nn_idx=nn_idx, k=k)
net = tf_util.conv2d(edge_feature, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='dgcnn1', bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net1 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k)
net = tf_util.conv2d(edge_feature, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='dgcnn2', bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net2 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k)
net = tf_util.conv2d(edge_feature, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='dgcnn3', bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net3 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=k)
edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k)
net = tf_util.conv2d(edge_feature, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='dgcnn4', bn_decay=bn_decay)
net = tf.reduce_max(net, axis=-2, keep_dims=True)
net4 = net
net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='agg', bn_decay=bn_decay)
net = tf.reduce_max(net, axis=1, keep_dims=True)
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" ConvNet baseline, input is BxNx9 gray image """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -1)
k = 20
weight_decay = 0 # 添加的
adj = tf_util.pairwise_distance(point_cloud[:, :,
6:]) # in:B*N*3 out:B*N*N
nn_idx = tf_util.knn(adj,
k=k) # (batch, num_points, k) # out:B*N*K
edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx,
k=k) # out:B*N*K*18
out1 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv1',
bn_decay=bn_decay,
is_dist=True)
out2 = tf_util.conv2d(out1,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv2',
bn_decay=bn_decay,
is_dist=True)
net_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)
adj = tf_util.pairwise_distance(net_1)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k)
out3 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv3',
bn_decay=bn_decay,
is_dist=True)
out4 = tf_util.conv2d(out3,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv4',
bn_decay=bn_decay,
is_dist=True)
net_2 = tf.reduce_max(out4, axis=-2, keep_dims=True)
adj = tf_util.pairwise_distance(net_2)
nn_idx = tf_util.knn(adj, k=k)
edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k)
out5 = tf_util.conv2d(edge_feature,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
weight_decay=weight_decay,
scope='adj_conv5',
bn_decay=bn_decay,
is_dist=True)
# out6 = tf_util.conv2d(out5, 64, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training, weight_decay=weight_decay,
# scope='adj_conv6', bn_decay=bn_decay, is_dist=True)
net_3 = tf.reduce_max(out5, axis=-2, keep_dims=True)
out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1),
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='adj_conv7',
bn_decay=bn_decay,
is_dist=True)
out_max = tf_util.max_pool2d(out7, [num_point, 1],
padding='VALID',
scope='maxpool')
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3])
# CONV
net = tf_util.conv2d(concat,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='seg/conv1',
is_dist=True)
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='seg/conv2',
is_dist=True)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.conv2d(net,
13, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
scope='seg/conv3',
is_dist=True)
net = tf.squeeze(net, [2])
return net
def get_model(point_input,
labels_pl,
cls_label,
is_training,
bn_decay=None,
cls_flag=0,
coarse_flag=0,
mmd_flag=0,
pfs_flag=False,
fully_concate=False):
""" Seg, input is BxNx6, output BxNx50 """
batch_size = point_input.get_shape()[0].value
num_point1 = point_input.get_shape()[1].value
num_point2 = int(np.floor(num_point1 / 4.0))
num_point3 = int(np.floor(num_point2 / 4.0))
# num_point4 = int(np.floor(num_point3 / 4.0))
end_points = {}
k = 10
pk = 10
de1_1 = 1
de1_2 = 2
de2_1 = 1
de2_2 = 2
de3_1 = 1
de3_2 = 2
cls_label_one_hot = tf.one_hot(cls_label,
depth=NUM_CATEGORIES,
on_value=1.0,
off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot,
[batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point1, 1])
point_input1 = point_input
point_cloud1 = tf.slice(point_input1, [0, 0, 0], [-1, -1, 3])
r1_11 = 0
r1_12 = 0.1
r1_21 = 0
r1_22 = 0.2
r2_11 = 0
r2_12 = 0.4
r2_21 = 0
r2_22 = 0.8
r3_11 = 0
r3_12 = 1.6
r3_21 = 0
r3_22 = 3.2
p1_1 = 0
p1_2 = 0.4
p2_1 = 0
p2_2 = 1.6
# activation_fn = tf.math.softplus
activation_fn = tf.nn.relu
##################################################################################################
# Hierarchy 1
point_input = tf.concat([point_input, cls_label_one_hot], axis=-1)
hie_matrix1 = tf.math.maximum(
tf.sqrt(tf_util.pairwise_distance(point_cloud1)), 1e-20)
# dist_matrix1_1 = hie_matrix1
# dist_matrix1_2 = hie_matrix1
dist_matrix1_1 = None
dist_matrix1_2 = None
adj_matrix = tf_util.pairwise_distance(point_cloud1)
net1_1 = pan_util.point_atrous_conv(point_input,
adj_matrix,
dist_matrix1_1,
k,
de1_1,
r1_11,
r1_12,
64,
scope='page1_1',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
adj_matrix = tf_util.pairwise_distance(net1_1)
net1_2 = pan_util.point_atrous_conv(net1_1,
adj_matrix,
dist_matrix1_2,
k,
de1_2,
r1_21,
r1_22,
64,
scope='page1_2',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
net = tf.squeeze(net1_2)
##################################################################################################
# Hierarchy 2
dist_threshold1 = False
net, p1_idx, _, point_cloud2 = pan_util.edge_preserve_sampling(
net,
point_cloud1,
num_point2,
hie_matrix1,
dist_threshold1,
pk,
1,
p1_1,
p1_2,
pfs_flag,
atrous_flag=False)
# point_cloud2 = gather_point(point_cloud1, p1_idx)
hie_matrix2 = tf.math.maximum(
tf.sqrt(tf_util.pairwise_distance(point_cloud2)), 1e-20)
# dist_matrix2_1 = hie_matrix2
# dist_matrix2_2 = hie_matrix2
dist_matrix2_1 = None
dist_matrix2_2 = None
adj_matrix = tf_util.pairwise_distance(net)
net2_1 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix2_1,
k,
de2_1,
r2_11,
r2_12,
128,
scope='page2_1',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
adj_matrix = tf_util.pairwise_distance(net2_1)
net2_2 = pan_util.point_atrous_conv(net2_1,
adj_matrix,
dist_matrix2_2,
k,
de2_2,
r2_21,
r2_22,
128,
scope='page2_2',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
net = tf.squeeze(net2_2)
##################################################################################################
# Hierarchy 3
dist_threshold2 = False
net, p2_idx, _, point_cloud3 = pan_util.edge_preserve_sampling(
net,
point_cloud2,
num_point3,
hie_matrix2,
dist_threshold2,
pk,
1,
p2_1,
p2_2,
pfs_flag,
atrous_flag=False)
# point_cloud3 = gather_point(point_cloud2, p2_idx)
hie_matrix3 = tf.math.maximum(
tf.sqrt(tf_util.pairwise_distance(point_cloud3)), 1e-20)
# dist_matrix3_1 = hie_matrix3
# dist_matrix3_2 = hie_matrix3
dist_matrix3_1 = None
dist_matrix3_2 = None
adj_matrix = tf_util.pairwise_distance(net)
net3_1 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix3_1,
k,
de3_1,
r3_11,
r3_12,
256,
scope='page3_1',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
adj_matrix = tf_util.pairwise_distance(net3_1)
net3_2 = pan_util.point_atrous_conv(net3_1,
adj_matrix,
dist_matrix3_2,
k,
de3_2,
r3_21,
r3_22,
256,
scope='page3_2',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
##################################################################################################
# Embeded Features
net = tf_util.conv2d(net3_2,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=activation_fn,
bn=True,
is_training=is_training,
scope='encoder',
bn_decay=bn_decay)
net = tf.reduce_max(net, axis=1, keepdims=True)
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
activation_fn=None,
scope='rg1',
bn_decay=bn_decay)
if mmd_flag > 0:
end_points['embedding'] = net
else:
end_points['embedding'] = None
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='rgdp1')
if cls_flag > 0:
cls_net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
activation_fn=activation_fn,
scope='cls_rg',
bn_decay=bn_decay)
cls_net = tf_util.dropout(cls_net,
keep_prob=0.5,
is_training=is_training,
scope='cls_rgdp')
cls_net = tf_util.fully_connected(cls_net,
NUM_CATEGORIES,
activation_fn=None,
scope='cls_fc')
end_points['cls_pred'] = cls_net
else:
end_points['cls_pred'] = None
global_feature_size = 1024
net = tf_util.fully_connected(net,
global_feature_size,
bn=True,
is_training=is_training,
activation_fn=activation_fn,
scope='rg2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='rgdp2')
net = tf.reshape(net, [batch_size, 1, 1, global_feature_size])
net = tf.tile(net, [1, num_point3, 1, 1])
net = tf.concat([net, net3_2], axis=-1)
net = tf_util.conv2d(net,
512, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=activation_fn,
bn=True,
is_training=is_training,
scope='decoder',
bn_decay=bn_decay)
if coarse_flag > 0:
coarse_net = tf.squeeze(net)
coarse_net = tf_util.conv1d(coarse_net,
128,
1,
padding='VALID',
bn=True,
activation_fn=activation_fn,
is_training=is_training,
scope='coarse_fc1',
bn_decay=bn_decay)
coarse_net = tf_util.dropout(coarse_net,
keep_prob=0.5,
is_training=is_training,
scope='cdp1')
coarse_net = tf_util.conv1d(coarse_net,
50,
1,
padding='VALID',
activation_fn=None,
scope='coarse_fc2')
coarse_labels_pl = tf_util.gather_labels(labels_pl, p1_idx)
coarse_labels_pl = tf_util.gather_labels(coarse_labels_pl, p2_idx)
end_points['coarse_pred'] = coarse_net
end_points['coarse_label'] = coarse_labels_pl
else:
end_points['coarse_pred'] = None
end_points['coarse_label'] = None
##################################################################################################
# Hierarchy 3
adj_matrix = tf_util.pairwise_distance(net)
net3_2 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix3_2,
k,
de3_2,
r3_21,
r3_22,
256,
scope='pagd3_2',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
net = tf.concat([net3_2, net3_1], axis=-1)
if fully_concate:
net3_2 = tf.squeeze(net3_2)
adj_matrix = tf_util.pairwise_distance(net)
net3_1 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix3_1,
k,
de3_1,
r3_11,
r3_12,
256,
scope='pagd3_1',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
net3_1 = tf.squeeze(net3_1)
##################################################################################################
# Hierarchy 2
idx, weight = pan_util.three_nn_upsampling(point_cloud2, point_cloud3)
net3_1 = three_interpolate(net3_1, idx, weight)
if fully_concate:
net3_2 = three_interpolate(net3_2, idx, weight)
net = tf.concat([tf.expand_dims(net3_1, 2), net2_2], axis=-1)
adj_matrix = tf_util.pairwise_distance(net)
net2_2 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix2_2,
k,
de2_2,
r2_21,
r2_22,
128,
scope='pagd2_2',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
net = tf.concat([net2_2, net2_1], axis=-1)
if fully_concate:
net2_2 = tf.squeeze(net2_2)
adj_matrix = tf_util.pairwise_distance(net)
net2_1 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix2_1,
k,
de2_1,
r2_11,
r2_12,
128,
scope='pagd2_1',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
net2_1 = tf.squeeze(net2_1)
##################################################################################################
# Hierarchy 1
idx, weight = pan_util.three_nn_upsampling(point_cloud1, point_cloud2)
net2_1 = three_interpolate(net2_1, idx, weight)
net3_1 = three_interpolate(net3_1, idx, weight)
if fully_concate:
net2_2 = three_interpolate(net2_2, idx, weight)
net3_2 = three_interpolate(net3_2, idx, weight)
net = tf.concat([tf.expand_dims(net2_1, 2), net1_2], axis=-1)
adj_matrix = tf_util.pairwise_distance(net)
net1_2 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix1_2,
k,
de1_2,
r1_21,
r1_22,
64,
scope='pagd1_2',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
net = tf.concat([net1_2, net1_1], axis=-1)
adj_matrix = tf_util.pairwise_distance(net)
net1_1 = pan_util.point_atrous_conv(net,
adj_matrix,
dist_matrix1_1,
k,
de1_1,
r1_11,
r1_12,
64,
scope='pagd1_1',
bn=True,
bn_decay=bn_decay,
is_training=is_training,
activation_fn=activation_fn)
##################################################################################################
# Final Prediction
if fully_concate:
net = tf.concat([
net1_1, net1_2,
tf.expand_dims(net2_1, 2),
tf.expand_dims(net2_2, 2),
tf.expand_dims(net3_1, 2),
tf.expand_dims(net3_2, 2)
],
axis=-1)
else:
net = tf.concat(
[net1_1,
tf.expand_dims(net2_1, 2),
tf.expand_dims(net3_1, 2)],
axis=-1)
net = tf.squeeze(net)
net = tf.concat([net, cls_label_one_hot], axis=-1)
net = tf_util.conv1d(net,
128,
1,
padding='VALID',
bn=True,
activation_fn=activation_fn,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloudz
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=1024,
radius=0.1,
nsample=32,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=256,
radius=0.2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=64,
radius=0.4,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=16,
radius=0.8,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Set abstraction layers
# Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4).
# So we only use NCHW for layer 1 until this issue can be resolved.
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=512,
radius=0.2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=128,
radius=0.4,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is B x N x 5, output B x num_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 2])
end_points['l0_xyz'] = l0_xyz
# c0
c0_l0_xyz, c0_l0_points, c0_l0_indices = pointSIFT_res_module(
l0_xyz,
l0_points,
radius=0.1,
out_channel=64,
is_training=is_training,
bn_decay=bn_decay,
scope='layer0_c0',
merge='concat')
l1_xyz, l1_points, l1_indices = pointnet_sa_module(c0_l0_xyz,
c0_l0_points,
npoint=1024,
radius=0.1,
nsample=32,
mlp=[64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
# c1
c0_l1_xyz, c0_l1_points, c0_l1_indices = pointSIFT_res_module(
l1_xyz,
l1_points,
radius=0.25,
out_channel=128,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1_c0')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(c0_l1_xyz,
c0_l1_points,
npoint=256,
radius=0.2,
nsample=32,
mlp=[128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
# c2
c0_l2_xyz, c0_l2_points, c0_l2_indices = pointSIFT_res_module(
l2_xyz,
l2_points,
radius=0.5,
out_channel=256,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2_c0')
c1_l2_xyz, c1_l2_points, c1_l2_indices = pointSIFT_res_module(
c0_l2_xyz,
c0_l2_points,
radius=0.5,
out_channel=512,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2_c1',
same_dim=True)
l2_cat_points = tf.concat([c0_l2_points, c1_l2_points], axis=-1)
fc_l2_points = tf_util.conv1d(l2_cat_points,
512,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='conv_2_fc',
bn_decay=bn_decay)
# c3
l3_xyz, l3_points, l3_indices = pointnet_sa_module(c1_l2_xyz,
fc_l2_points,
npoint=64,
radius=0.4,
nsample=32,
mlp=[512, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [512, 512],
is_training,
bn_decay,
scope='fa_layer2')
_, l2_points_1, _ = pointSIFT_module(l2_xyz,
l2_points,
radius=0.5,
out_channel=512,
is_training=is_training,
bn_decay=bn_decay,
scope='fa_layer2_c0')
_, l2_points_2, _ = pointSIFT_module(l2_xyz,
l2_points,
radius=0.5,
out_channel=512,
is_training=is_training,
bn_decay=bn_decay,
scope='fa_layer2_c1')
_, l2_points_3, _ = pointSIFT_module(l2_xyz,
l2_points,
radius=0.5,
out_channel=512,
is_training=is_training,
bn_decay=bn_decay,
scope='fa_layer2_c2')
l2_points = tf.concat([l2_points_1, l2_points_2, l2_points_3], axis=-1)
l2_points = tf_util.conv1d(l2_points,
512,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fa_2_fc',
bn_decay=bn_decay)
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer3')
_, l1_points_1, _ = pointSIFT_module(l1_xyz,
l1_points,
radius=0.25,
out_channel=256,
is_training=is_training,
bn_decay=bn_decay,
scope='fa_layer3_c0')
_, l1_points_2, _ = pointSIFT_module(l1_xyz,
l1_points_1,
radius=0.25,
out_channel=256,
is_training=is_training,
bn_decay=bn_decay,
scope='fa_layer3_c1')
l1_points = tf.concat([l1_points_1, l1_points_2], axis=-1)
l1_points = tf_util.conv1d(l1_points,
256,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fa_1_fc',
bn_decay=bn_decay)
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat([l0_xyz, l0_points], axis=-1),
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
_, l0_points, _ = pointSIFT_module(l0_xyz,
l0_points,
radius=0.1,
out_channel=128,
is_training=is_training,
bn_decay=bn_decay,
scope='fa_layer4_c0')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = tf_util.conv2d(net_transformed,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
input_image = tf.expand_dims(point_cloud, -1)
# Point functions (MLP implemented as conv2d)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# Symmetric function: max pooling. tf_util.max_pool2d is DEPRECATED:
# net = tf_util.max_pool2d(net, [num_point,1],
# padding='VALID', scope='maxpool')
net = tf.keras.layers.MaxPooling2D(net, [num_point, 1],
padding='VALID',
scope='maxpool')
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_second_model(point_cloud, input_label, is_training, cat_num, part_num, \
batch_size, num_point, weight_decay, bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
end_points = {}
with tf.variable_scope('transform_net1') as sc:
K = 3
transform = get_transform(point_cloud, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
out1 = tf_util.conv2d(input_image,
64, [1, K],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
out1_max = tf_util.max_pool2d(out1, [num_point, 1],
padding='VALID',
scope='maxpool1')
out1_max = tf.tile(out1_max, [1, num_point, 1, 1])
out2 = tf_util.conv2d(out1,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
out2_max = tf_util.max_pool2d(out2, [num_point, 1],
padding='VALID',
scope='maxpool2')
out2_max = tf.tile(out2_max, [1, num_point, 1, 1])
out3 = tf_util.conv2d(out2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
out3_max = tf_util.max_pool2d(out3, [num_point, 1],
padding='VALID',
scope='maxpool3')
out3_max = tf.tile(out3_max, [1, num_point, 1, 1])
with tf.variable_scope('transform_net2') as sc:
K = 128
transform = get_transform_K(out3, is_training, bn_decay, K)
end_points['transform'] = transform
squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128])
net_transformed = tf.matmul(squeezed_out3, transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net_transformed_max = tf_util.max_pool2d(net_transformed, [num_point, 1],
padding='VALID',
scope='maxpool4')
net_transformed_max = tf.tile(net_transformed_max, [1, num_point, 1, 1])
out4 = tf_util.conv2d(net_transformed,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
out5 = tf_util.conv2d(out4,
2048, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
out_max = tf_util.max_pool2d(out5, [num_point, 1],
padding='VALID',
scope='maxpool')
# classification network
net = tf.reshape(out_max, [batch_size, -1])
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='cla/fc1',
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='cla/fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='cla/dp1')
net = tf_util.fully_connected(net,
cat_num,
activation_fn=None,
scope='cla/fc3')
# segmentation network
one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num])
out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand])
expand = tf.tile(out_max, [1, num_point, 1, 1])
#concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5])#modify by zgh
concat = tf.concat(axis=3,
values=[expand, out1, out2, out3, out1_max, out4, out5])
net2 = tf_util.conv2d(concat,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv1',
weight_decay=weight_decay)
net2 = tf_util.dropout(net2,
keep_prob=0.8,
is_training=is_training,
scope='seg/dp1')
net2 = tf_util.conv2d(net2,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv2',
weight_decay=weight_decay)
net2 = tf_util.dropout(net2,
keep_prob=0.8,
is_training=is_training,
scope='seg/dp2')
net2 = tf_util.conv2d(net2,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv3',
weight_decay=weight_decay)
net2 = tf_util.conv2d(net2,
part_num, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
scope='seg/conv4',
weight_decay=weight_decay)
net2 = tf.reshape(net2, [batch_size, num_point, part_num])
return net, net2, end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx4, onehotvec is Bx3, output BxNx2 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
input_image = tf.expand_dims(point_cloud, -1)
net = tf_util.conv2d(input_image, 64, [1,6],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
point_feat = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(point_feat, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
global_feat = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
print global_feat
global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3)
print 'Global Feat: ', global_feat
global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])
print point_feat, global_feat_expand
concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand])
print concat_feat
net = tf_util.conv2d(concat_feat, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv6', bn_decay=bn_decay)
net = tf_util.conv2d(net, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv7', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv8', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv9', bn_decay=bn_decay)
net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5)
logits = tf_util.conv2d(net, 2, [1,1],
padding='VALID', stride=[1,1], activation_fn=None,
scope='conv10')
logits = tf.squeeze(logits, [2]) # BxNxC
print logits
print '-----------'
#net = tf.concat(axis=3, values=[net, tf.expand_dims(tf.slice(point_cloud, [0,0,0], [-1,-1,3]), 2)])
mask = tf.slice(logits,[0,0,0],[-1,-1,1]) < tf.slice(logits,[0,0,1],[-1,-1,1])
mask = tf.to_float(mask) # BxNx1
mask_count = tf.tile(tf.reduce_sum(mask,axis=1,keep_dims=True), [1,1,3]) # Bx1x3
print mask
point_cloud_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) # BxNx3
# ---- Subtract points mean ----
mask_xyz_mean = tf.reduce_sum(tf.tile(mask, [1,1,3])*point_cloud_xyz, axis=1, keep_dims=True) # Bx1x3
mask_xyz_mean = mask_xyz_mean/tf.maximum(mask_count,1) # Bx1x3
point_cloud_xyz_stage1 = point_cloud_xyz - tf.tile(mask_xyz_mean, [1,num_point,1])
print 'Point cloud xyz stage1: ', point_cloud_xyz_stage1
# ---- Regress 1st stage center ----
net = tf.expand_dims(point_cloud_xyz_stage1, 2)
print net
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg1-stage1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg2-stage1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg3-stage1', bn_decay=bn_decay)
mask_expand = tf.tile(tf.expand_dims(mask,-1), [1,1,1,256])
masked_net = net*mask_expand
print masked_net
net = tf_util.max_pool2d(masked_net, [num_point,1], padding='VALID', scope='maxpool-stage1')
net = tf.squeeze(net, axis=[1,2])
print net
net = tf.concat([net, one_hot_vec], axis=1)
net = tf_util.fully_connected(net, 256, scope='fc1-stage1', bn=True, is_training=is_training, bn_decay=bn_decay)
net = tf_util.fully_connected(net, 128, scope='fc2-stage1', bn=True, is_training=is_training, bn_decay=bn_decay)
stage1_center = tf_util.fully_connected(net, 3, activation_fn=None, scope='fc3-stage1')
stage1_center = stage1_center + tf.squeeze(mask_xyz_mean, axis=1) # Bx3
end_points['stage1_center'] = stage1_center
# ---- Subtract stage1 center ----
point_cloud_xyz_submean = point_cloud_xyz - tf.expand_dims(stage1_center, 1)
print 'Point cloud xyz submean: ', point_cloud_xyz_submean
net = tf.expand_dims(point_cloud_xyz_submean, 2)
print net
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg2', bn_decay=bn_decay)
net = tf_util.conv2d(net, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg4', bn_decay=bn_decay)
mask_expand = tf.tile(tf.expand_dims(mask,-1), [1,1,1,512])
masked_net = net*mask_expand
print masked_net
net = tf_util.max_pool2d(masked_net, [num_point,1], padding='VALID', scope='maxpool2')
net = tf.squeeze(net, axis=[1,2])
print net
net = tf.concat([net, one_hot_vec], axis=1)
net = tf_util.fully_connected(net, 512, scope='fc1', bn=True, is_training=is_training, bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, scope='fc2', bn=True, is_training=is_training, bn_decay=bn_decay)
# First 3 are cx,cy,cz, next NUM_HEADING_BIN*2 are for heading
# next NUM_SIZE_CLUSTER*4 are for dimension
output = tf_util.fully_connected(net, 3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4, activation_fn=None, scope='fc3')
print output
center = tf.slice(output, [0,0], [-1,3])
center = center + stage1_center # Bx3
end_points['center'] = center
heading_scores = tf.slice(output, [0,3], [-1,NUM_HEADING_BIN])
heading_residuals_normalized = tf.slice(output, [0,3+NUM_HEADING_BIN], [-1,NUM_HEADING_BIN])
end_points['heading_scores'] = heading_scores # BxNUM_HEADING_BIN
end_points['heading_residuals_normalized'] = heading_residuals_normalized # BxNUM_HEADING_BIN (should be -1 to 1)
end_points['heading_residuals'] = heading_residuals_normalized * (np.pi/NUM_HEADING_BIN) # BxNUM_HEADING_BIN
size_scores = tf.slice(output, [0,3+NUM_HEADING_BIN*2], [-1,NUM_SIZE_CLUSTER]) # BxNUM_SIZE_CLUSTER
size_residuals_normalized = tf.slice(output, [0,3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER], [-1,NUM_SIZE_CLUSTER*3])
size_residuals_normalized = tf.reshape(size_residuals_normalized, [batch_size, NUM_SIZE_CLUSTER, 3]) # BxNUM_SIZE_CLUSTERx3
end_points['size_scores'] = size_scores
end_points['size_residuals_normalized'] = size_residuals_normalized
end_points['size_residuals'] = size_residuals_normalized * tf.expand_dims(tf.constant(mean_size_arr, dtype=tf.float32), 0)
return logits, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=512,
radius=0.2,
nsample=64,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=128,
radius=0.4,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """
batch_size = point_cloud.shape[0]
num_point = point_cloud.shape[1]
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=512,
radius=0.2,
nsample=64,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=128,
radius=0.4,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat([l0_xyz, l0_points], axis=-1),
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer3')
end_points['feats_0'] = l0_points
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net # 我觉得这里就算feature map了，但是一个点只用了128个元素来描述了
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def pointnet_sa_module(cascade_id,
xyz,
points,
bidmap,
mlp_configs,
block_bottom_center_mm,
configs,
sgf_config_pls,
is_training,
bn_decay,
scope,
bn=True,
tnet_spec=None,
use_xyz=True,
IsShowModel=False):
'''
Input cascade_id==0:
xyz is grouped_points: (batch_size,nsubblock0,npoint_subblock0,6)
points: None
bidmap: None
Input cascade_id==1:
xyz: (batch_size,nsubblock0,3)
points: (batch_size,nsubblock0,channel)
bidmap: (batch_size,nsubblock1,npoint_subblock1)
Medium cascade_id==1:
grouped_xyz: (batch_size,nsubblock1,npoint_subblock1,3)
new_xyz: (batch_size,nsubblock1,3)
group_points: (batch_size,nsubblock1,npoint_subblock1,channel)
output cascade_id==1:
new_xyz: (batch_size,nsubblock1,3)
new_points: (batch_size,nsubblock1,channel)
'''
block_bottom_center_mm = tf.cast(
block_bottom_center_mm, tf.float32, name='block_bottom_center_mm'
) # gpu_0/sa_layer3/block_bottom_center_mm:0
batch_size = xyz.get_shape()[0].value
with tf.variable_scope(scope) as sc:
cascade_num = configs['flatten_bm_extract_idx'].shape[
0] - 1 # include global here (Note: cascade_num does not include global in block_pre_util )
assert configs['sub_block_step_candis'].size == cascade_num - 1
if cascade_id == 0:
indrop_keep_mask = tf.get_default_graph().get_tensor_by_name(
'indrop_keep_mask:0') # indrop_keep_mask:0
assert len(xyz.shape) == 3
if bidmap == None:
grouped_xyz = tf.expand_dims(xyz, 1)
grouped_points = tf.expand_dims(points, 1)
new_xyz = None
valid_mask = None
else:
batch_idx = tf.reshape(tf.range(batch_size), [batch_size, 1, 1, 1])
nsubblock = bidmap.get_shape()[1].value
npoint_subblock = bidmap.get_shape()[2].value
batch_idx_ = tf.tile(batch_idx, [1, nsubblock, npoint_subblock, 1])
bidmap = tf.expand_dims(bidmap, axis=-1, name='bidmap')
bidmap_concat = tf.concat(
[batch_idx_, bidmap], axis=-1,
name='bidmap_concat') # gpu_0/sa_layer0/bidmap_concat:0
# The value for invalid item in bidmap is -17.
# On GPU, the responding grouped_xyz and grouped_points is 0.
# NOT WORK on CPU !!!
# invalid indices comes from merge_blocks_while_fix_bmap
# set point_indices_f for invalid points as
# NETCONFIG['redundant_points_in_block'] ( shoud be set < -500)
valid_mask = tf.greater(bidmap, tf.constant(
-500, tf.int32), 'valid_mask') # gpu_0/sa_layer0/valid_mask:0
grouped_xyz = tf.gather_nd(
xyz, bidmap_concat,
name='grouped_xyz') # gpu_0/sa_layer0/grouped_xyz:0
grouped_points = tf.gather_nd(points,
bidmap_concat,
name='group_points')
if cascade_id == 0 and len(indrop_keep_mask.get_shape()) != 0:
grouped_indrop_keep_mask = tf.gather_nd(
indrop_keep_mask,
bidmap_concat,
name='grouped_indrop_keep_mask'
) # gpu_0/sa_layer0/grouped_indrop_keep_mask:0
# new_xyz is the "voxel center" or "mean position of points in the voxel"
if configs['mean_grouping_position'] and (
not mlp_configs['block_learning'] == '3DCNN'):
new_xyz = tf.reduce_mean(grouped_xyz, -2)
else:
new_xyz = block_bottom_center_mm[:, :, 3:6] * tf.constant(
0.001, tf.float32)
# the mid can be mean or block center, decided by configs['mean_grouping_position']
sub_block_mid = tf.expand_dims(
new_xyz, -2, name='sub_block_mid') # gpu_1/sa_layer0/sub_block_mid
global_block_mid = tf.reduce_mean(sub_block_mid,
1,
keepdims=True,
name='global_block_mid')
grouped_xyz_submid = grouped_xyz - sub_block_mid
grouped_xyz_glomid = grouped_xyz - global_block_mid
grouped_xyz_feed = []
if 'raw' in configs['xyz_elements']:
grouped_xyz_feed.append(grouped_xyz)
if 'sub_mid' in configs['xyz_elements']:
grouped_xyz_feed.append(grouped_xyz_submid)
if 'global_mid' in configs['xyz_elements']:
grouped_xyz_feed.append(grouped_xyz_glomid)
grouped_xyz_feed = tf.concat(grouped_xyz_feed, -1)
if cascade_id == 0:
# xyz must be at the first in feed_data_elements !!!!
grouped_points = tf.concat(
[grouped_xyz_feed, grouped_points[..., 3:]], -1)
if len(indrop_keep_mask.get_shape()) != 0:
if InDropMethod == 'set1st':
# set all the dropped item as the first item
tmp1 = tf.multiply(grouped_points,
grouped_indrop_keep_mask)
points_1st = grouped_points[:, :, 0:1, :]
points_1st = tf.tile(points_1st,
[1, 1, grouped_points.shape[2], 1])
indrop_mask_inverse = 1 - grouped_indrop_keep_mask
tmp2 = indrop_mask_inverse * points_1st
grouped_points = tf.add(
tmp1, tmp2, name='grouped_points_droped'
) # gpu_0/sa_layer0/grouped_points_droped
#tf.add_to_collection( 'check', grouped_points )
elif InDropMethod == 'set0':
valid_mask = tf.logical_and(
valid_mask,
tf.equal(grouped_indrop_keep_mask, 0),
name='valid_mask_droped'
) # gpu_1/sa_layer0/valid_mask_droped
elif use_xyz:
grouped_points = tf.concat([grouped_xyz_feed, grouped_points],
axis=-1)
tf.add_to_collection('grouped_xyz', grouped_xyz)
tf.add_to_collection('grouped_xyz_submid', grouped_xyz_submid)
tf.add_to_collection('grouped_xyz_glomid', grouped_xyz_glomid)
if cascade_id > 0 and use_xyz and (not cascade_id == cascade_num - 1):
grouped_points = tf.concat([grouped_xyz_feed, grouped_points],
axis=-1)
nsample = grouped_points.get_shape()[2].value # the conv kernel size
if IsShowModel:
print(
'\n\npointnet_sa_module cascade_id:%d\n xyz:%s\n grouped_xyz:%s\n new_xyz:%s\n grouped_points:%s\n nsample:%d'
% (cascade_id, shape_str([xyz]), shape_str([grouped_xyz]),
shape_str([new_xyz]), shape_str([grouped_points]), nsample))
new_points = grouped_points
if valid_mask != None:
new_points = new_points * tf.cast(valid_mask[:, :, :, 0:1],
tf.float32)
if 'growth_rate' in mlp_configs['point_encoder'][cascade_id]:
new_points = tf_util.dense_net( new_points, mlp_configs['point_encoder'][cascade_id], bn, is_training, bn_decay,\
scope = 'dense_cascade_%d_point_encoder'%(cascade_id) , is_show_model = IsShowModel )
else:
for i, num_out_channel in enumerate(
mlp_configs['point_encoder'][cascade_id]):
new_points = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv%d' % (i),
bn_decay=bn_decay)
if configs['Cnn_keep_prob'] < 1:
if (not configs['only_last_layer_ineach_cascade']
) or i == len(
mlp_configs['point_encoder'][cascade_id]) - 1:
new_points = tf_util.dropout(
new_points,
keep_prob=configs['Cnn_keep_prob'],
is_training=is_training,
scope='dropout',
name='cnn_dp%d' % (i))
if IsShowModel:
print('point encoder1 %d, new_points:%s' %
(i, shape_str([new_points])))
if cascade_id == 0:
root_point_features = new_points
#if InDropMethod == 'set0':
# if len(indrop_keep_mask.get_shape()) != 0:
# new_points = tf.identity(new_points,'points_before_droped') # gpu_0/sa_layer0/points_before_droped:0
# new_points = tf.multiply( new_points, grouped_indrop_keep_mask, name='droped_points' ) # gpu_0/sa_layer0/droped_points:0
else:
root_point_features = None
pooling = mlp_configs['block_learning']
if pooling == '3DCNN' and (cascade_id == 0):
pooling = 'max'
#if pooling=='avg':
# new_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1')
#elif pooling=='weighted_avg':
# with tf.variable_scope('weighted_avg1'):
# dists = tf.norm(grouped_xyz,axis=-1,ord=2,keep_dims=True)
# exp_dists = tf.exp(-dists * 5)
# weights = exp_dists/tf.reduce_sum(exp_dists,axis=2,keep_dims=True) # (batch_size, npoint, nsample, 1)
# new_points *= weights # (batch_size, npoint, nsample, mlps_0[-1])
# new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True)
if pooling == 'max':
# Even the grouped_points and grouped_xyz are 0 for invalid points, the
# vaule after mlp will not be. It has to be set as 0 forcely before
# pooling.
if valid_mask != None:
new_points = new_points * tf.cast(valid_mask[:, :, :, 0:1],
tf.float32)
new_points = tf.identity(
new_points,
'points_before_max') # gpu_0/sa_layer0/points_before_max
new_points = tf.reduce_max(new_points,
axis=[2],
keepdims=True,
name='points_after_max')
#elif pooling=='min':
# new_points = tf_util.max_pool2d(-1*new_points, [1,nsample], stride=[1,1], padding='VALID', scope='minpool1')
#elif pooling=='max_and_avg':
# avg_points = tf_util.max_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='maxpool1')
# max_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1')
# new_points = tf.concat([avg_points, max_points], axis=-1)
elif pooling == '3DCNN':
new_points = grouped_points_to_voxel_points(
cascade_id,
new_points,
valid_mask,
block_bottom_center_mm,
configs,
grouped_xyz,
IsShowVoxelModel=IsShowModel)
if IsShowModel:
print('voxel points:%s' % (shape_str([new_points])))
for i, num_out_channel in enumerate(
mlp_configs['voxel_channels'][cascade_id]):
kernel_i = [mlp_configs['voxel_kernels'][cascade_id][i]] * 3
stride_i = [mlp_configs['voxel_strides'][cascade_id][i]] * 3
if new_points.shape[1] % 2 == 0:
padding_i = np.array([[0, 0], [1, 0], [1, 0], [1, 0], [
0, 0
]]) * mlp_configs['voxel_paddings'][cascade_id][i]
else:
padding_i = np.array([[0, 0], [1, 1], [1, 1], [1, 1], [
0, 0
]]) * mlp_configs['voxel_paddings'][cascade_id][i]
new_points = tf.pad(new_points, padding_i, "CONSTANT")
if type(num_out_channel) == int:
new_points = tf_util.conv3d(new_points,
num_out_channel,
kernel_i,
scope='3dconv_%d' % (i),
stride=stride_i,
padding='VALID',
bn=bn,
is_training=is_training,
bn_decay=bn_decay,
name='points_3dcnn_%d' % (i))
if IsShowModel:
print('block learning by 3dcnn %d, new_points:%s' %
(i, shape_str([new_points])))
elif num_out_channel == 'max':
new_points = tf_util.max_pool3d(new_points,
kernel_i,
scope='3dmax_%d' % (i),
stride=stride_i,
padding='VALID')
if IsShowModel:
print('block learning max pooling %d, new_points:%s' %
(i, shape_str([new_points])))
elif num_out_channel == 'avg':
new_points = tf_util.avg_pool3d(new_points,
kernel_i,
scope='3dmax_%d' % (i),
stride=stride_i,
padding='VALID')
if IsShowModel:
print('block learning avg pooling %d, new_points:%s' %
(i, shape_str([new_points])))
# gpu_0/sa_layer1/3dconv_0/points_3dcnn_0:0
if configs['Cnn_keep_prob'] < 1:
if (not configs['only_last_layer_ineach_cascade']
) or i == len(
mlp_configs['voxel_channels'][cascade_id]) - 1:
new_points = tf_util.dropout(
new_points,
keep_prob=configs['Cnn_keep_prob'],
is_training=is_training,
scope='dropout',
name='3dcnn_dp%d' % (i))
# gpu_0/sa_layer4/3dconv_0/points_3dcnn_0:0
new_points = tf.squeeze(new_points, [1, 2, 3])
new_points = tf.reshape(
new_points, [batch_size, -1, 1, new_points.shape[-1].value])
if IsShowModel:
print('after %s, new_points:%s' %
(pooling, shape_str([new_points])))
if 'growth_rate' in mlp_configs['block_encoder'][cascade_id]:
new_points = tf_util.dense_net(
new_points,
mlp_configs['block_encoder'][cascade_id],
bn,
is_training,
bn_decay,
scope='dense_cascade_%d_block_encoder' % (cascade_id),
is_show_model=IsShowModel)
else:
for i, num_out_channel in enumerate(
mlp_configs['block_encoder'][cascade_id]):
new_points = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv_post_%d' % (i),
bn_decay=bn_decay)
if configs['Cnn_keep_prob'] < 1:
if (not configs['only_last_layer_ineach_cascade']
) or i == len(
mlp_configs['block_encoder'][cascade_id]) - 1:
new_points = tf_util.dropout(
new_points,
keep_prob=configs['Cnn_keep_prob'],
is_training=is_training,
scope='dropout',
name='cnn_dp%d' % (i))
if IsShowModel:
print('block encoder %d, new_points:%s' %
(i, shape_str([new_points])))
# (2, 512, 1, 64)
new_points = tf.squeeze(new_points,
[2]) # (batch_size, npoints, mlps_1[-1])
if IsShowModel:
print(
'pointnet_sa_module return\n new_xyz: %s\n new_points:%s\n\n' %
(shape_str([new_xyz]), shape_str([new_points])))
#import pdb;pdb.set_trace()
# (2, 512, 64)
return new_xyz, new_points, root_point_features
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx5, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 2])
end_points['l0_xyz'] = l0_xyz
print(l0_xyz.shape)
print(l0_points.shape)
# Layer 1
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
1024, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay=bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
256, [0.1, 0.3, 0.5], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
128,
radius=0.6,
nsample=64,
mlp=[256, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
16,
radius=0.8,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat([l0_xyz, l0_points], axis=-1),
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(video,
is_training,
num_classes=400,
bn_decay=0.999,
weight_decay=0.0001,
pool_t=False,
pool_first=False,
freeze_bn=False):
""" Video Net, input is BxTxHxWx3, output Bx400 """
bsize = video.get_shape()[0].value
end_points = {}
channel_stride = [(64, 1), (128, 2), (256, 2), (512, 2)]
# res block options
num_blocks = [3, 4, 23, 3]
# pointnet options
topks = [None, 4, 4, None]
shrink_ratios = [None, 2, None, None]
pn = [[0] * 3, [0, 0, 1, 1], [0] * 20 + [1] * 3, [0] * 3]
net = tf_util.conv3d(
video,
64, [1, 7, 7],
stride=[1, 2 if pool_first else 1, 2 if pool_first else 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training,
weight_decay=weight_decay,
freeze_bn=freeze_bn,
scope='conv0')
net = tf_util.max_pool3d(net, [1, 3, 3],
stride=[1, 2, 2],
scope='pool0',
padding='SAME')
for gp, cs in enumerate(channel_stride):
n_channels = cs[0]
stride = cs[1]
with tf.variable_scope('group{}'.format(gp)):
for i in range(num_blocks[gp]):
with tf.variable_scope('block{}'.format(i)):
end_points['res{}_{}_in'.format(gp, i)] = net
if i == 0:
net_bra = tf_util.conv3d(net, n_channels, [1, 1, 1], stride=[1, stride, stride], bn=True, bn_decay=bn_decay, \
is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1')
else:
net_bra = tf_util.conv3d(net, n_channels, [1, 1, 1], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \
is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1')
net_bra = tf_util.conv3d(net_bra, n_channels, [1, 3, 3], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \
is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv2')
net_bra = tf_util.conv3d(net_bra, n_channels * 4, [1, 1, 1], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \
is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv3')
if net.get_shape()[-1].value != (n_channels * 4):
net = tf_util.conv3d(net, n_channels * 4, [1, 1, 1], stride=[1, stride, stride], bn=True, bn_decay=bn_decay, \
is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='convshortcut')
net = net + net_bra
end_points['res{}_{}_mid'.format(gp, i)] = net
if pn[gp][i]:
c = net.get_shape()[-1].value
net_pointnet, end_point = net_utils.cp_module(net, k=topks[gp], mlp0=[c//8], mlp=[c//8], scope='pointnet', is_training=is_training, bn_decay=bn_decay, \
weight_decay=weight_decay, distance='l2', activation_fn=None, freeze_bn=freeze_bn, shrink_ratio=shrink_ratios[gp])
net += net_pointnet
end_points['pointnet{}_{}'.format(gp, i)] = end_point
end_points['after_pointnet{}_{}'.format(gp, i)] = net
net = tf.nn.relu(net)
end_points['res{}_{}_out'.format(gp, i)] = net
net = tf.reduce_mean(net, [1, 2, 3])
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp')
net = tf_util.fully_connected(net,
num_classes,
activation_fn=None,
weight_decay=weight_decay,
scope='fc')
return net, end_points
def get_instance_seg_v1_net_modified(point_cloud, one_hot_vec,
is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v1 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
net = tf.expand_dims(point_cloud, 2)
c1 = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
c1_mp = tf_util.max_pool2d(c1, [num_point,1], padding='VALID', scope='maxpool1')
c1_mp = tf.tile(c1_mp, [1, num_point, 1, 1])
c1 = tf.concat(axis=3, values=[c1, c1_mp])
c2 = tf_util.conv2d(c1, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
c2_mp = tf_util.max_pool2d(c2, [num_point,1], padding='VALID', scope='maxpool2')
c2_mp = tf.tile(c2_mp, [1, num_point, 1, 1])
c2 = tf.concat(axis=3, values=[c2, c2_mp])
c3 = tf_util.conv2d(c2, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
c3_mp = tf_util.max_pool2d(c3, [num_point,1], padding='VALID', scope='maxpool3')
c3_mp = tf.tile(c3_mp, [1, num_point, 1, 1])
c3 = tf.concat(axis=3, values=[c3, c3_mp])
c4 = tf_util.conv2d(c3, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
c4_mp = tf_util.max_pool2d(c4, [num_point,1], padding='VALID', scope='maxpool4')
c4_mp = tf.tile(c4_mp, [1, num_point, 1, 1])
c4 = tf.concat(axis=3, values=[c4, c4_mp])
c5 = tf_util.conv2d(c4, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
global_feat = tf_util.max_pool2d(c5, [num_point,1],
padding='VALID', scope='maxpool')
global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3)
global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])
# concat_feat = tf.concat(axis=3, values=[c3, global_feat_expand])
global_feat_expand = tf.concat(axis=3, values=[c5, global_feat_expand])
d1 = tf_util.conv2d(global_feat_expand, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv6', bn_decay=bn_decay)
d1 = tf.concat(axis=3, values=[d1, c4])
d2 = tf_util.conv2d(d1, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv7', bn_decay=bn_decay)
d2 = tf.concat(axis=3, values=[d2, c3])
d3 = tf_util.conv2d(d2, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv8', bn_decay=bn_decay)
d3 = tf.concat(axis=3, values=[d3, c2])
d4 = tf_util.conv2d(d3, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv9', bn_decay=bn_decay)
d4 = tf.concat(axis=3, values=[d4, c1])
net = tf_util.dropout(d4, is_training, 'dp1', keep_prob=0.5)
logits = tf_util.conv2d(net, 2, [1,1],
padding='VALID', stride=[1,1], activation_fn=None,
scope='conv10')
logits = tf.squeeze(logits, [2]) # BxNxC
return logits, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \
batch_size, num_point, weight_decay, bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
end_points = {}
with tf.variable_scope('transform_net1') as sc:
K = 3
transform = get_transform(point_cloud, is_training, bn_decay, K = 3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
out1 = tf_util.conv2d(input_image, 64, [1,K], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay)
out2 = tf_util.conv2d(out1, 128, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay)
out3 = tf_util.conv2d(out2, 128, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
K = 128
transform = get_transform_K(out3, is_training, bn_decay, K)
end_points['transform'] = transform
squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128])
net_transformed = tf.matmul(squeezed_out3, transform)
net_transformed = tf.expand_dims(net_transformed, [2])
out4 = tf_util.conv2d(net_transformed, 512, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay)
out5 = tf_util.conv2d(out4, 2048, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay)
out_max = tf_util.max_pool2d(out5, [num_point,1], padding='VALID', scope='maxpool')
# classification network
net = tf.reshape(out_max, [batch_size, -1])
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1')
net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3')
# segmentation network
one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num])
out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand])
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5])
net2 = tf_util.conv2d(concat, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay,
bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay)
net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1')
net2 = tf_util.conv2d(net2, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay,
bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay)
net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2')
net2 = tf_util.conv2d(net2, 128, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay,
bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay)
net2 = tf_util.conv2d(net2, part_num, [1,1], padding='VALID', stride=[1,1], activation_fn=None,
bn=False, scope='seg/conv4', weight_decay=weight_decay)
net2 = tf.reshape(net2, [batch_size, num_point, part_num])
return net, net2, end_points
def get_model(point_cloud, is_training, bn_decay=None, output_dim=1):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
# num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp2')
# net = tf_util.fully_connected(net, 1, activation_fn=None, scope='fc3')
net = tf_util.fully_connected(net,
output_dim,
activation_fn=None,
scope='fc3')
net = tf.squeeze(tf.sigmoid(net))
return net, end_points
def get_model(point_cloud,
is_training,
num_neighbors,
farthest_distance,
bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx9, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud[:, :, 0:3]
l0_points = point_cloud[:, :, 3:9]
"""Point Enrichment """
new_xyz = l0_xyz # (batch_size, npoint, 3)
idx, pts_cnt = query_ball_point(farthest_distance, num_neighbors, l0_xyz,
new_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
neighbor_xyz -= tf.tile(tf.expand_dims(new_xyz, 2),
[1, 1, num_neighbors, 1])
neighbor_points = group_point(l0_points, idx)
neighbor_representation = tf.concat([neighbor_xyz, neighbor_points],
axis=-1)
neighbor_representation = tf.reshape(neighbor_representation,
(batch_size, num_point, -1))
num_channel = neighbor_representation.get_shape()[2].value
points = tf_util.conv1d(point_cloud,
num_channel,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='points_fc',
bn_decay=bn_decay)
neighbor_representation_gp = gating_process(
neighbor_representation,
num_channel,
padding='VALID',
is_training=is_training,
scope='neighbor_representation_gp',
bn_decay=bn_decay)
points_gp = gating_process(points,
num_channel,
padding='VALID',
is_training=is_training,
scope='points_gp',
bn_decay=bn_decay)
l0_points = tf.concat([
neighbor_representation_gp * points,
points_gp * neighbor_representation
],
axis=-1)
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module_withgab(
l0_xyz,
l0_points,
npoint=1024,
radius=0.1,
nsample=32,
mlp=[32, 32, 64],
mlp2=[64, 64],
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
gab=True)
l2_xyz, l2_points, l2_indices = pointnet_sa_module_withgab(
l1_xyz,
l1_points,
npoint=256,
radius=0.2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module_withgab(
l2_xyz,
l2_points,
npoint=64,
radius=0.4,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module_withgab(
l3_xyz,
l3_points,
npoint=16,
radius=0.8,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
"""Spatial-wise Attention"""
input = net
output_a = tf_util.conv2d(tf.expand_dims(input, 1),
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_a',
bn_decay=bn_decay)
output_b = tf_util.conv2d(tf.expand_dims(input, 1),
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_b',
bn_decay=bn_decay)
output_b = tf.transpose(output_b, [0, 1, 3, 2])
output_a = tf.squeeze(output_a)
output_b = tf.squeeze(output_b)
energy = tf.matmul(output_a, output_b)
attention = tf.nn.softmax(energy, axis=-1)
output_d = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_d',
bn_decay=bn_decay)
output_d = tf.squeeze(output_d)
gamma = tf_util._variable_with_weight_decay('weight_patial',
shape=[1],
use_xavier=True,
stddev=1e-3,
wd=0.0)
output_SA = tf.matmul(attention, output_d)
output_SA = output_SA * gamma + tf.squeeze(input)
"""Channel-wise Attention"""
output_f = tf.transpose(input, [0, 2, 1])
energy = tf.matmul(output_f, input)
D = tf.reduce_max(energy, -1)
D = tf.expand_dims(D, -1)
energy_new = tf.tile(D, multiples=[1, 1, energy.shape[2]]) - energy
attention = tf.nn.softmax(energy_new, axis=-1)
output_CA = tf.matmul(input, attention)
gamma2 = tf_util._variable_with_weight_decay('weightsgamma2m',
shape=[1],
use_xavier=True,
stddev=1e-3,
wd=0.0)
output_CA = output_CA * gamma2 + input
output = output_SA + output_CA
end_points['feats'] = output
net = tf_util.dropout(output,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
13,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def pointnet_fp_module(cascade_id,
num_neighbors,
points1,
points2,
flatten_bidxmap,
fbmap_neighbor_idis,
mlps_e1,
mlps_fp,
is_training,
bn_decay,
scope,
configs,
bn=True,
IsShowModel=False):
'''
in Qi's code, 3 larger balls are weighted back-propogated to one point
Here, I only back-propogate one
Input:
points1 (cascade_id=2): (2, 256, 256)
points2 (cascade_id=3): (2, 64, 512)
flatten_bidxmap: (B,num_point,self.flatbxmap_max_nearest_num,2)
[:,:,:,0]: aim_b_index
[:,:,:,1]: point_index_in_aimb (useless when cascade_id>0)
fbmap_neighbor_idis: (B,num_point,self.flatbxmap_max_nearest_num,1)
[:,:,:,2]: index_distance
mlps_fp: [256,256]
Output:
new_points1: (2, 256, 256)
'''
if IsShowModel:
print(
'\n\npointnet_fp_module %s\n points1: %s\n points2: %s\n flatten_bidxmap: %s\n'
% (scope, shape_str([points1]), shape_str(
[points2]), shape_str([flatten_bidxmap])))
with tf.variable_scope(scope) as sc:
assert len(flatten_bidxmap.get_shape()) == 4
if cascade_id == 0:
# points1 is grouped point features
assert len(points1.get_shape()) == 4
else:
# points1 is flat point features
assert len(points1.get_shape()) == 3
assert flatten_bidxmap.shape[1] == points1.shape[1]
batch_size = points2.get_shape()[0].value
batch_idx0 = tf.reshape(tf.range(batch_size), [batch_size, 1, 1, 1])
point1_num = flatten_bidxmap.get_shape()[1].value
if cascade_id == 0:
# convert grouped points1 to flat point features
#flatten_bidxmap_aimbidx1 = flatten_bidxmap[:,:,0,0:2] # (2, 256, 1)
#flatten_bidxmap_aimbidx_concat1 = tf.concat( [batch_idx, flatten_bidxmap_aimbidx1], axis=-1 )
#points1 = tf.gather_nd(points1, flatten_bidxmap_aimbidx_concat1 )
num_neighbor0 = num_neighbors[0]
disw_theta0 = -1.5 # the abs smaller, more smooth
assert num_neighbor0 <= flatten_bidxmap.shape[2].value
batch_idx = tf.tile(
batch_idx0, [1, point1_num, num_neighbor0, 1]) # (2, 256, 1)
flatten_bidxmap_concat1 = tf.concat(
[batch_idx, flatten_bidxmap[:, :, 0:num_neighbor0, 0:2]],
axis=-1) # [...,[batch_idx,aimb_idx,point_idx_in_aimb] ]
points1_nei = tf.gather_nd(points1, flatten_bidxmap_concat1)
if num_neighbor0 > 1:
dis_weight = tf.nn.softmax(
fbmap_neighbor_idis[:, :, 0:num_neighbor0, :] *
disw_theta0,
axis=2)
points1_nei = tf.multiply(points1_nei, dis_weight)
points1 = tf.reduce_sum(points1_nei, axis=2)
#flatten_bidxmap_aimbidx = flatten_bidxmap[:,:,0,0:1] # (2, 256, 1)
#flatten_bidxmap_aimbidx_concat = tf.concat( [batch_idx, flatten_bidxmap_aimbidx],axis=-1 ) # (2, 256, 2)
#mapped_points2 = tf.gather_nd(points2, flatten_bidxmap_aimbidx_concat) # (2, 256, 512)
# use the inverse distance weighted sum of 3 neighboured point features
if cascade_id == 0:
num_neighbor = num_neighbors[1]
else:
num_neighbor = num_neighbors[2]
assert num_neighbor <= flatten_bidxmap.shape[2].value
neighbor_method = 'A'
#-----------------------------------
if neighbor_method == 'A':
batch_idx = tf.tile(batch_idx0,
[1, point1_num, 1, 1]) # (2, 256, 1)
# from distance to weight
if num_neighbor > 1:
disw_theta = -0.5 # the abs smaller, more smooth
dis_weight = tf.nn.softmax(fbmap_neighbor_idis * disw_theta,
axis=2)
for i in range(num_neighbor):
flatten_bidxmap_aimbidx_concat_i = tf.concat(
[batch_idx, flatten_bidxmap[:, :, i:(i + 1), 0:1]],
axis=-1) # (2, 256, 2)
mapped_points2_nei_i = tf.gather_nd(
points2, flatten_bidxmap_aimbidx_concat_i) # (2, 256, 512)
if num_neighbor > 1:
mapped_points2_nei_i = tf.multiply(
mapped_points2_nei_i, dis_weight[:, :, i:(i + 1), :])
if i == 0:
mapped_points2 = mapped_points2_nei_i
else:
mapped_points2 += mapped_points2_nei_i
else:
mapped_points2 = mapped_points2_nei_i
mapped_points2 = tf.squeeze(mapped_points2, 2)
#-----------------------------------
if neighbor_method == 'B':
batch_idx = tf.tile(
batch_idx0, [1, point1_num, num_neighbor, 1]) # (2, 256, 1)
flatten_bidxmap_aimbidx_concat = tf.concat(
[batch_idx, flatten_bidxmap[:, :, 0:num_neighbor, 0:1]],
axis=-1) # (2, 256, 2)
mapped_points2_nei = tf.gather_nd(
points2, flatten_bidxmap_aimbidx_concat) # (2, 256, 512)
if num_neighbor > 1:
disw_theta = -0.7 # the abs smaller, more smooth
dis_weight = tf.nn.softmax(fbmap_neighbor_idis * disw_theta,
axis=2)
mapped_points2_nei = tf.multiply(mapped_points2_nei,
dis_weight)
mapped_points2 = tf.reduce_sum(mapped_points2_nei, 2)
#-----------------------------------
new_points1 = points1
new_points1 = tf.expand_dims(new_points1, 1)
if 'growth_rate' in mlps_e1:
new_points1 = tf_util.dense_net(new_points0,
mlps_e1,
bn,
is_training,
bn_decay,
scope='dense_cascade_%d_fp' %
(cascade_id),
is_show_model=IsShowModel)
else:
for i, num_out_channel in enumerate(mlps_e1):
new_points1 = tf_util.conv2d(new_points1,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv_encoder1_%d' % (i),
bn_decay=bn_decay)
if configs['Cnn_keep_prob'] < 1:
if (not configs['only_last_layer_ineach_cascade']
) or i == len(mlps_e1) - 1:
new_points1 = tf_util.dropout(
new_points1,
keep_prob=configs['Cnn_keep_prob'],
is_training=is_training,
scope='dropout',
name='cnn_dp%d' % (i))
if IsShowModel:
print('new_points1:%s' % (shape_str([new_points1])))
mapped_points2 = tf.expand_dims(mapped_points2, 1)
new_points1 = tf.concat(values=[new_points1, mapped_points2], axis=-1)
if IsShowModel:
print('after concat new_points1:%s' % (shape_str([new_points1])))
if 'growth_rate' in mlps_fp:
new_points1 = tf_util.dense_net(new_points1,
mlps_fp,
bn,
is_training,
bn_decay,
scope='dense_cascade_%d_fp' %
(cascade_id),
is_show_model=IsShowModel)
else:
for i, num_out_channel in enumerate(mlps_fp):
new_points1 = tf_util.conv2d(new_points1,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv%d' % (i),
bn_decay=bn_decay)
if configs['Cnn_keep_prob'] < 1:
if (not configs['only_last_layer_ineach_cascade']
) or i == len(mlps_fp) - 1:
new_points1 = tf_util.dropout(
new_points1,
keep_prob=configs['Cnn_keep_prob'],
is_training=is_training,
scope='dropout',
name='cnn_dp%d' % (i))
if IsShowModel:
print('new_points1:%s' % (shape_str([new_points1])))
new_points1 = tf.squeeze(new_points1, [1]) # (2, 256, 256)
if IsShowModel: print('new_points1:%s' % (shape_str([new_points1])))
#if IsShowModel: import pdb; pdb.set_trace()
return new_points1
