def get_model(point_cloud, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def up_sample_folding_tile(x, ratio, grid_2d, h, w, scope, bn_decay,
weight_decay, is_training):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc:
batch_size = x.get_shape()[0].value
npoint = x.get_shape()[1].value
ch = x.get_shape()[2].value
x = tf.tile(tf.reshape(x, [batch_size, npoint, 1, ch]),
[1, 1, ratio, 1])
x = tf.reshape(x, [batch_size, npoint * ratio, ch])
x = add_2d_grid_tile(x,
grid_2d,
h,
w,
scope=('add_grid_%sx%s' % (h, w)))
x_3 = tf_util.conv1d(x,
ch,
kernel_size=1,
stride=1,
scope='upsample_conv3',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay)
x_3 = tf_util.conv1d(x,
ch / 2,
kernel_size=1,
stride=1,
scope='upsample_conv2',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay)
x_3 = tf_util.conv1d(x,
3,
kernel_size=1,
stride=1,
scope='upsample_conv0',
bn=False,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
activation_fn=None)
tf.add_to_collection('foldings', x_3)
x = tf.concat([x, x_3], -1)
x = self_attention(x,
scope='upsample_self_attention',
bn_decay=bn_decay,
weight_decay=weight_decay,
is_training=is_training)
x = tf_util.conv1d(x,
ch,
kernel_size=1,
stride=1,
scope='upsample_conv',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay)
return x
def vote_layer(xyz, points, mlp_list, is_training, bn_decay, bn, scope):
"""
Voting layer
"""
with tf.variable_scope(scope) as sc:
for i, channel in enumerate(mlp_list):
points = tf_util.conv1d(points,
channel,
1,
padding='VALID',
stride=1,
bn=bn,
scope='vote_layer_%d' % i,
bn_decay=bn_decay,
is_training=is_training)
ctr_offsets = tf_util.conv1d(points,
3,
1,
padding='VALID',
stride=1,
bn=False,
activation_fn=None,
scope='vote_offsets')
min_offset = tf.reshape(cfg.MODEL.MAX_TRANSLATE_RANGE, [1, 1, 3])
limited_ctr_offsets = tf.minimum(tf.maximum(ctr_offsets, min_offset),
-min_offset)
xyz = xyz + limited_ctr_offsets
return xyz, points, ctr_offsets
def pt_last(inputs, scope, out_dim, bn_decay, is_training):
"""
:param inputs: NHC
:param scope:
:param out_dim:
:param bn_decay:
:param is_training:
:return:
"""
x = tf_util.conv1d(inputs, out_dim, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope=scope + 'conv0', bn_decay=bn_decay)
# NHC (32, 256, 256)
x = tf_util.conv1d(x, out_dim, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope=scope + 'conv1', bn_decay=bn_decay)
x1 = sa(x, out_dim, scope + "1", bn_decay, is_training)
x2 = sa(x1, out_dim, scope + "2", bn_decay, is_training)
x3 = sa(x2, out_dim, scope + "3", bn_decay, is_training)
x4 = sa(x3, out_dim, scope + "4", bn_decay, is_training)
# concat in C (NHC) axis
x = tf.concat([x1, x2, x3, x4], axis=-1)
return x
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def pointnet_sa_module(xyz, points, mlp,
is_training, bn_decay, bn,
scope):
''' PointNet Set Abstraction (SA) Module (Last Layer)
Sample all points within the point cloud and extract a global feature
Input:
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor
mlp_list: list of int32 -- output size for MLP on each point
Return:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor
idx: (batch_size, npoint, nsample) int32 -- indices for local regions
'''
with tf.variable_scope(scope) as sc:
grouped_points = tf.concat([xyz, points], axis=-1) # [bs, npoint, 3+c]
for j, num_out_channel in enumerate(mlp):
grouped_points = tf_util.conv1d(grouped_points,
num_out_channel,
1,
padding='VALID',
bn=bn,
is_training=is_training,
scope='conv%d' % j,
bn_decay=bn_decay)
# bs, num_out_channel
new_points = tf.reduce_max(grouped_points, axis=1)
return new_points
def self_attention(x, scope, bn_decay, weight_decay, is_training):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc:
batch_size = x.get_shape()[0].value
npoint = x.get_shape()[1].value
ch = x.get_shape()[2].value
f = tf_util.conv1d(x,
ch // 8,
kernel_size=1,
stride=1,
scope='f_conv',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
activation_fn=None)
g = tf_util.conv1d(x,
ch // 8,
kernel_size=1,
stride=1,
scope='g_conv',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay,
activation_fn=None)
h = tf_util.conv1d(x,
ch,
kernel_size=1,
stride=1,
scope='h_conv',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay)
s = tf.matmul(g, f, transpose_b=True) # (batch_size, npoint, npoint)
attention_map = tf.nn.softmax(s)
o = tf.matmul(attention_map, h) # (batch_size, npoint, ch)
gamma = tf.get_variable("gamma", [1],
initializer=tf.constant_initializer(0.0))
o = o * gamma + x
return o
def sa(inputs, out_dim, scope, bn_decay, is_training):
# NHC
x_q = tf_util.conv1d(inputs, out_dim // 4, kernel_size=1,
padding='VALID', stride=1, bn=False,
is_biases=False, is_training=is_training,
activation_fn=None, scope=scope + 'q')
# NHC
x_k = tf_util.conv1d(inputs, out_dim // 4, kernel_size=1,
padding='VALID', stride=1, bn=False,
is_biases=False, is_training=is_training,
activation_fn=None, scope=scope + 'k')
# NCH
x_k = tf.transpose(x_k, [0, 2, 1])
# NHC
x_v = tf_util.conv1d(inputs, out_dim, kernel_size=1,
padding='VALID', stride=1,
bn=False, is_training=is_training,
activation_fn=None, scope=scope + 'v')
# NCH
x_v = tf.transpose(x_v, [0, 2, 1])
energy = tf.matmul(x_q, x_k)
attention = tf.nn.softmax(energy, axis=- 1)
attention = attention / (1e-9 + tf.reduce_sum(attention, axis=1, keepdims=True))
# NCH
x_r = tf.matmul(x_v, attention)
# NHC
x_r = tf.transpose(x_r, [0, 2, 1])
x_r = tf_util.conv1d(inputs - x_r, out_dim, kernel_size=1,
padding='VALID', stride=1,
bn=True, is_training=is_training,
scope=scope + 'attention', bn_decay=bn_decay)
x = inputs + x_r
return x
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = tf.shape(point_cloud)[0]
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4, 0.8], [64, 128],
[[128, 128, 256], [128, 196, 256]], is_training, bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None,
mlp=[256, 512, 1024], mlp2=None, group_all=True,
is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay,
scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points], axis=-1),
l1_points, [128, 128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def down_sample(x, ratio, scope, bn_decay, weight_decay, is_training):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc:
batch_size = x.get_shape()[0].value
npoint = x.get_shape()[1].value
ch = x.get_shape()[2].value
x = tf.reshape(x, [batch_size, int(npoint / ratio), -1])
x = tf_util.conv1d(x,
ch,
kernel_size=1,
stride=1,
scope='downsample_conv',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay)
return x
def attention(x, y, scope, bn_decay, weight_decay, is_training):
"""
Args:
x: (batch_size, npoint1, ch1)
y: (batch_size, npoint2, ch2)
Returns:
o: (batch_size, npoint2, ch2)
"""
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc:
batch_size = x.get_shape()[0].value
npoint1 = x.get_shape()[1].value
ch1 = x.get_shape()[2].value
npoint2 = y.get_shape()[1].value
ch2 = y.get_shape()[2].value
dims = int(min(ch1, ch2) // 4)
bi_ins = tf.matmul(y, tf.transpose(x, perm=[0, 2, 1]))
cos_y = tf.multiply(y, y)
cos_y = tf.reduce_sum(cos_y, 2, keep_dims=True)
cos_y = tf.sqrt(cos_y)
cos_y = tf.add(cos_y, 1e-7)
cos_x = tf.multiply(x, x)
cos_x = tf.reduce_sum(cos_x, 2, keep_dims=True)
cos_x = tf.sqrt(cos_x)
cos_x = tf.add(cos_x, 1e-7)
dis = tf.matmul(cos_y, tf.transpose(cos_x, perm=[0, 2, 1]))
bi_ins = tf.div(bi_ins, dis)
o = tf.matmul(bi_ins, x)
o = tf_util.conv1d(o,
ch2,
kernel_size=1,
stride=1,
scope='x_conv',
bn=True,
is_training=is_training,
bn_decay=bn_decay,
weight_decay=weight_decay)
gamma = tf.get_variable("gamma", [1],
initializer=tf.constant_initializer(0.0))
o = o * gamma + y
return o
def get_model(radius,
layer,
point_cloud,
is_training,
bn_decay=None,
knn=False,
flow_module='default'):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
end_points = {}
batch_size = point_cloud.get_shape()[0].value # batch_size = 16
num_point = point_cloud.get_shape()[1].value // 2
# change here, num_point hard coded to 2048
# num_point = 2048
l0_xyz_f1 = point_cloud[:, :num_point, 0:3]
l0_points_f1 = point_cloud[:, :num_point, 3:]
l0_xyz_f2 = point_cloud[:, num_point:, 0:3]
l0_points_f2 = point_cloud[:, num_point:, 3:]
RADIUS1 = 0.5
RADIUS2 = 1.0
RADIUS3 = 2.0
RADIUS4 = 4.0
with tf.variable_scope('sa1') as scope:
# radius, npoints, nlayers, mlp size, sampling technique
# Set conv layers, POINT FEATURE LEARNING
# Frame 1, Layer 1 (with radius = 0.5)
l1_xyz_f1, l1_points_f1, l1_indices_f1 = pointnet_sa_module(
l0_xyz_f1,
l0_points_f1,
npoint=1024,
radius=RADIUS1,
nsample=16,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
knn=knn)
end_points['l1_indices_f1'] = l1_indices_f1
end_points['l1_xyz_f1'] = l1_points_f1
end_points['l1_input_f1'] = l0_xyz_f1
# Frame 1, Layer 2 (with radius = 1.0), Inputs are the above function's output
l2_xyz_f1, l2_points_f1, l2_indices_f1 = pointnet_sa_module(
l1_xyz_f1,
l1_points_f1,
npoint=256,
radius=RADIUS2,
nsample=16,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2',
knn=knn)
end_points['l2_indices_f1'] = l2_indices_f1
end_points['l2_xyz_f1'] = l2_points_f1
end_points['l2_input_f1'] = l1_xyz_f1
scope.reuse_variables()
# Frame 2, Layer 1 (with radius = 0.5)
l1_xyz_f2, l1_points_f2, l1_indices_f2 = pointnet_sa_module(
l0_xyz_f2,
l0_points_f2,
npoint=1024,
radius=RADIUS1,
nsample=16,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
knn=knn)
end_points['l1_points_f2'] = l1_points_f2
end_points['l1_xyz_f2'] = l1_indices_f2
end_points['l1_input_f2'] = l0_xyz_f2
# Tensor("sa1/layer1_1/GatherPoint:0", shape=(16, 1024, 3), dtype=float32, device= / device: GPU:0)
# Tensor("sa1/layer1_1/Squeeze:0", shape=(16, 1024, 64), dtype=float32, device= / device: GPU:0)
# Tensor("sa1/layer1_1/QueryBallPoint:0", shape=(16, 1024, 16), dtype=int32, device= / device: GPU:0)
# Frame 2, Layer 2(with radius = 1.0), input are of the above function's output
l2_xyz_f2, l2_points_f2, l2_indices_f2 = pointnet_sa_module(
l1_xyz_f2,
l1_points_f2,
npoint=256,
radius=RADIUS2,
nsample=16,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2',
knn=knn)
end_points['l2_points_f2'] = l2_points_f2
end_points['l2_xyz_f2'] = l2_indices_f2
end_points['l2_input_f2'] = l1_xyz_f2
# Tensor("sa1/layer2_1/GatherPoint:0", shape=(16, 256, 3), dtype=float32, device= / device: GPU:0)
# Tensor("sa1/layer2_1/Squeeze:0", shape=(16, 256, 128), dtype=float32, device= / device: GPU:0)
# Tensor("sa1/layer2_1/QueryBallPoint:0", shape=(16, 256, 16), dtype=int32, device= / device: GPU:0)
# POINT MIXTURE
# embedding layer
# radius = 1, 10, 50
print("Radius here:", radius)
print('KNN', knn)
print('flow module', flow_module)
if flow_module == 'default':
_, l2_points_f1_new = flow_embedding_module(l2_xyz_f1,
l2_xyz_f2,
l2_points_f1,
l2_points_f2,
radius=radius,
nsample=64,
mlp=[128, 128, 128],
is_training=is_training,
bn_decay=bn_decay,
scope='flow_embedding',
bn=True,
pooling='max',
knn=True,
corr_func='concat')
end_points['l2_points_f1_new'] = l2_points_f1_new
elif flow_module == 'all':
_, l2_points_f1_new = flow_embedding_module_all(
l2_xyz_f1,
l2_xyz_f2,
l2_points_f1,
l2_points_f2,
radius=radius,
nsample=256,
mlp=[128, 128, 128],
is_training=is_training,
bn_decay=bn_decay,
scope='flow_embedding',
bn=True,
pooling='max',
knn=True,
corr_func='concat')
end_points['l2_points_f1_new'] = l2_points_f1_new
# setconv layer
# Layer 3 with radius = 2.0
l3_xyz_f1, l3_points_f1, l3_indices_f1 = pointnet_sa_module(
l2_xyz_f1,
l2_points_f1_new,
npoint=64,
radius=RADIUS3,
nsample=8,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
end_points['l3_indices_f1'] = l3_indices_f1
end_points['l3_xyz_f1'] = l3_points_f1
# Tensor("layer3/GatherPoint:0", shape=(16, 64, 3), dtype=float32, device=/device:GPU:0)
# Tensor("layer3/Squeeze:0", shape=(16, 64, 256), dtype=float32, device=/device:GPU:0)
# Tensor("layer3/QueryBallPoint:0", shape=(16, 64, 8), dtype=int32, device=/device:GPU:0)
# Layer 4 with radius = 4.0
l4_xyz_f1, l4_points_f1, l4_indices_f1 = pointnet_sa_module(
l3_xyz_f1,
l3_points_f1,
npoint=16,
radius=RADIUS4,
nsample=8,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
end_points['l4_indices_f1'] = l4_indices_f1
end_points['l4_xyz_f1'] = l4_points_f1
# Tensor("layer4/GatherPoint:0", shape=(16, 16, 3), dtype=float32, device=/device:GPU:0)
# Tensor("layer4/Squeeze:0", shape=(16, 16, 512), dtype=float32, device=/device:GPU:0)
# Tensor("layer4/QueryBallPoint:0", shape=(16, 16, 8), dtype=int32, device=/device:GPU:0)
### FLOW REFINEMENT MODULE
# Feature Propagation
# Frame 1, l1->l2; l2->l3; l3->l4
l3_feat_f1 = set_upconv_module(l3_xyz_f1,
l4_xyz_f1,
l3_points_f1,
l4_points_f1,
nsample=8,
radius=2.4,
mlp=[],
mlp2=[256, 256],
scope='up_sa_layer1',
is_training=is_training,
bn_decay=bn_decay,
knn=True)
end_points['l3_feat_f1'] = l3_feat_f1
l2_feat_f1 = set_upconv_module(
l2_xyz_f1,
l3_xyz_f1,
tf.concat(axis=-1, values=[l2_points_f1, l2_points_f1_new]),
l3_feat_f1,
nsample=8,
radius=1.2,
mlp=[128, 128, 256],
mlp2=[256],
scope='up_sa_layer2',
is_training=is_training,
bn_decay=bn_decay,
knn=True)
end_points['l2_feat_f1'] = l2_feat_f1
l1_feat_f1 = set_upconv_module(l1_xyz_f1,
l2_xyz_f1,
l1_points_f1,
l2_feat_f1,
nsample=8,
radius=0.6,
mlp=[128, 128, 256],
mlp2=[256],
scope='up_sa_layer3',
is_training=is_training,
bn_decay=bn_decay,
knn=True)
end_points['l1_feat_f1'] = l1_feat_f1
if layer == 'pointnet':
l0_feat_f1 = pointnet_fp_module(l0_xyz_f1,
l1_xyz_f1,
l0_points_f1,
l1_feat_f1, [256, 256],
is_training,
bn_decay,
scope='fa_layer4')
else:
print('Last set conv layer running')
l0_feat_f1 = set_upconv_module(l0_xyz_f1,
l1_xyz_f1,
l0_points_f1,
l1_feat_f1,
nsample=8,
radius=0.3,
mlp=[128, 128, 256],
mlp2=[256],
scope='up_sa_layer4',
is_training=is_training,
bn_decay=bn_decay,
knn=True)
end_points['l0_feat_f1'] = l0_feat_f1
# FC layers
net = tf_util.conv1d(l0_feat_f1,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['net1'] = net
net = tf_util.conv1d(net,
3,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
end_points['net'] = net
return net, end_points
def body(global_idx, scale, l0_xyz, l0_points, bradius, gt, *args):
scale_idx = int(np.log(scale)/np.log(step_ratio)) - 1
gt_original = gt
with tf.variable_scope("level_%d" % (scale_idx+1), reuse=tf.AUTO_REUSE):
num_point_unaltered = l0_xyz.shape[1]
if scale_idx > 0:
l0_features = None if not use_l0_points else l0_points
l0_xyz, l0_features, gt = crop_input_and_gt(l0_xyz, l0_features, gt, scale)
if gt is not None:
gt = tf.stop_gradient(gt)
else:
gt = gt_original
l0_xyz_normalized, centroid, furthest_distance = tf_util.normalize_point_cloud(l0_xyz)
bradius = tf.constant(1.0, shape=[l0_xyz.shape[0]], dtype=tf.float32)
if not use_l0_points:
l0_features = tf.expand_dims(l0_xyz_normalized, axis=2)
if len(l0_features.shape) == 3:
l0_features = tf.expand_dims(l0_features, axis=2)
else:
l0_features = tf.expand_dims(l0_xyz, axis=2)
l0_xyz_normalized = l0_xyz
num_point = l0_xyz.get_shape()[1].value or max_num_point
l0_features = tf_util.conv2d(l0_features, 24, [1, 1],
padding='VALID', scope='layer0', is_training=is_training, bn=use_bn, ibn=use_ibn, bn_decay=bn_decay, activation_fn=None)
l0_features = tf.squeeze(l0_features, axis=2)
# encoding layer
l1_features, l1_idx = tf_util.dense_conv1(l0_features, growth_rate=growth_rate, n=dense_n, k=knn,
scope="layer1", is_training=is_training, bn=use_bn, ibn=use_ibn, bn_decay=bn_decay)
l1_features = tf.concat([l1_features, l0_features], axis=-1) # (12+24*2)+24=84
l2_features = tf_util.conv1d(l1_features, comp, 1, # 24
padding='VALID', scope='layer2_prep', is_training=is_training, bn=use_bn, ibn=use_ibn, bn_decay=bn_decay)
l2_features, l2_idx = tf_util.dense_conv1(l2_features, growth_rate=growth_rate, n=dense_n, k=knn,
scope="layer2", is_training=is_training, bn=use_bn, bn_decay=bn_decay)
l2_features = tf.concat([l2_features, l1_features], axis=-1) # 84+(24*2+12)=144
l3_features = tf_util.conv1d(l2_features, comp, 1, # 48
padding='VALID', scope='layer3_prep', is_training=is_training, bn=use_bn, ibn=use_ibn, bn_decay=bn_decay) # 48
l3_features, l3_idx = tf_util.dense_conv1(l3_features, growth_rate=growth_rate, n=dense_n, k=knn,
scope="layer3", is_training=is_training, bn=use_bn, bn_decay=bn_decay)
l3_features = tf.concat([l3_features, l2_features], axis=-1) # 144+(24*2+12)=204
l4_features = tf_util.conv1d(l3_features, comp, 1, # 48
padding='VALID', scope='layer4_prep', is_training=is_training, bn=use_bn, ibn=use_ibn, bn_decay=bn_decay) # 48
l4_features, l3_idx = tf_util.dense_conv1(l4_features, growth_rate=growth_rate, n=dense_n, k=knn,
scope="layer4", is_training=is_training, bn=use_bn, bn_decay=bn_decay)
l4_features = tf.concat([l4_features, l3_features], axis=-1) # 204+(24*2+12)=264
l4_features = tf.expand_dims(l4_features, axis=2)
if scale_idx > 0:
with tf.name_scope("skip_connection"):
lower_scale_idx = scale_idx - 1
skip_feature = tf.get_collection("SKIP_FEATURES_%d_%d" % (global_idx, lower_scale_idx))
skip_feature_xyz = tf.get_collection("SKIP_FEATURE_XYZ_%d_%d" % (global_idx, lower_scale_idx))
assert(len(skip_feature_xyz) == 1)
skip_feature_xyz = skip_feature_xyz.pop()
skip_feature = skip_feature.pop()
if not is_training and skip_feature_xyz.shape[0] != l0_xyz.shape[0]:
skip_feature_xyz = tf.tile(skip_feature_xyz, [l0_xyz.shape[0].value, 1, 1])
skip_feature = tf.tile(skip_feature, [l0_xyz.shape[0].value, 1, 1, 1])
if fm_knn > 1:
# find closest k point in spatial
dist, idx = knn_point(fm_knn, skip_feature_xyz, l0_xyz, sort=True, unique=True)
knn_feature = tf.gather_nd(tf.squeeze(skip_feature, axis=2), idx) # B, N, k, C
knn_xyz = tf.gather_nd(skip_feature_xyz, idx) # B, N, k,
# compute distance in feature and sptial sapce
# knn_feature = tf.stop_gradient(knn_feature)
d, f_average_weight = exponential_distance(l4_features, knn_feature, scope="feature_matching_fweight")
tf.contrib.summary.histogram("f_average_distance_{}_{}".format(scale_idx, lower_scale_idx), d)
d, s_average_weight = exponential_distance(tf.expand_dims(l0_xyz, axis=2), knn_xyz, scope="feature_matching_sweight")
tf.contrib.summary.histogram("s_average_distance_{}_{}".format(scale_idx, lower_scale_idx), d)
s_average_weight = tf.stop_gradient(s_average_weight)
f_average_weight = tf.stop_gradient(f_average_weight)
# weights: B, P, K, 1
tf.contrib.summary.histogram("f_average_weight_{}_{}".format(scale_idx, lower_scale_idx), f_average_weight)
tf.contrib.summary.histogram("s_average_weight_{}_{}".format(scale_idx, lower_scale_idx), s_average_weight)
average_weight = (f_average_weight * s_average_weight)
# average_weight = s_average_weight
average_weight = average_weight / tf.reduce_sum(average_weight+1e-5, axis=2, keepdims=True)
knn_feature = tf.reduce_sum(average_weight * knn_feature, axis=2, keepdims=True)
else:
dist, idx = knn_point(1, skip_feature_xyz, l0_xyz, sort=True, unique=True)
knn_feature = tf.gather_nd(tf.squeeze(skip_feature, axis=2), idx) # B, N, 1, C
l4_features = 0.2*knn_feature + l4_features
tf.add_to_collection("SKIP_FEATURE_XYZ_%d_%d" % (global_idx, scale_idx),
tf.concat(tf.split(l0_xyz, l0_xyz.shape[0]//batch_size, axis=0), axis=1))
tf.add_to_collection("SKIP_FEATURES_%d_%d" % (global_idx, scale_idx),
tf.concat(tf.split(l4_features, l4_features.shape[0]//batch_size, axis=0), axis=1))
with tf.variable_scope('up_layer', reuse=reuse):
if not np.isscalar(bradius):
bradius_expand = tf.expand_dims(tf.expand_dims(bradius, axis=-1), axis=-1)
else:
bradius_expand = bradius
if step_ratio < 4:
grid = gen_1d_grid(step_ratio)
expansion_ratio = step_ratio
else:
grid = gen_grid(np.round(np.sqrt(step_ratio)).astype(np.int32))
expansion_ratio = (np.round(np.sqrt(step_ratio))**2).astype(np.int32)
num_point = tf.shape(l0_xyz)[1]
grid = tf.tile(tf.expand_dims(grid, 0), [l0_xyz.shape[0], num_point, 1]) # [batch_size, num_point*4, 2])
grid = tf.expand_dims(grid*bradius_expand, axis=2)
# [B, N, 1, 1, 256+3] -> [B, N, 4, 1, 256+3] -> [B, num_point*4, 1, 256+3]
new_feature = tf.reshape(
tf.tile(tf.expand_dims(l4_features, 2), [1, 1, expansion_ratio, 1, 1]), [l4_features.shape[0], num_point*expansion_ratio, 1, l4_features.shape[-1]])
new_feature = tf.concat([new_feature, grid], axis=-1)
new_feature = tf_util.conv2d(new_feature, 128, [1, 1],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='up_layer1', bn_decay=bn_decay)
new_feature = tf_util.conv2d(new_feature, 128, [1, 1],
padding='VALID', stride=[1, 1],
bn=use_bn, is_training=is_training,
scope='up_layer2',
bn_decay=bn_decay)
# get the xyz
new_xyz = tf_util.conv2d(new_feature, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='fc_layer1', bn_decay=bn_decay)
new_xyz = tf_util.conv2d(new_xyz, 3, [1, 1],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='fc_layer2', bn_decay=bn_decay,
activation_fn=None, weight_decay=0.0) # B*(2N)*1*3
new_xyz = tf.squeeze(new_xyz, axis=2) # B*(2N)*3
new_feature = tf.squeeze(new_feature, axis=2) # B*(2N)*3
if not no_res:
new_xyz += tf.reshape(tf.tile(tf.expand_dims(l0_xyz_normalized, 2), [1, 1, expansion_ratio, 1]), [l0_xyz_normalized.shape[0], num_point*expansion_ratio, -1]) # B, N, 4, 3
if scale_idx > 0:
new_xyz = (new_xyz * furthest_distance) + centroid
if not is_training and (new_xyz.shape[0] != batch_size):
new_xyz = tf.concat(tf.split(new_xyz, new_xyz.shape[0]//batch_size, axis=0), axis=1)
output_num = num_point_unaltered*step_ratio
# resample to get sparser points idx [B, P, 1]
idx = tf.expand_dims(farthest_point_sample(output_num, new_xyz), axis=-1)
batch_indices = tf.tile(tf.reshape(tf.range(batch_size), (-1, 1, 1)), (1, output_num, 1))
new_xyz = tf.gather_nd(new_xyz, tf.concat([batch_indices, idx], axis=-1))
new_feature = tf.gather_nd(new_feature, tf.concat([batch_indices, idx], axis=-1))
bradius = furthest_distance
return new_xyz, new_feature, bradius, gt, l0_xyz
def densepoint_module(xyz,
features,
is_training,
bn_decay,
scope=None,
bn=True,
npoint=None,
radius=None,
nsample=None,
mlp=None,
ppool=None,
use_xyz=True,
group_num=1):
""" DensePoint module with PPool, Enhanced PConv and Global Pooling
Input:
xyz: (batch_size, npoint, 3) TF tensor
features: (batch_size, npoint, channel) TF tensor
npoint: int32 -- #points sampled in farthest point sampling
radius: float32 -- search radius in local region
nsample: int32 -- how many points selected in each local region
(if selected point less than expected, duplicate the selected)
mlp: int32 -- output size for SLP on each point
use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features
use_nchw: bool, if True, use NCHW data format for conv2d, which is usually faster than NHWC format
Return:
new_xyz: (batch_size, npoint, 3) TF tensor -centroid points
For PPool and Global Pooling:
new_features: (batch_size, npoint, Cin+3) TF tensor
For enhancedPConv:
all_new_features : (batch_size, npoint, cin+3+Cout/4)
"""
with tf.compat.v1.variable_scope(scope) as sc:
if npoint is not None:
new_xyz, grouped_features = sample_and_group(
npoint, radius, nsample, xyz, features, use_xyz)
if ppool: # ppool
grouped_features = tf_util.batch_norm_for_conv2d(
grouped_features,
is_training=is_training,
scope='BNConcat',
bn_decay=bn_decay)
new_grouped_features = tf_util.conv2d(grouped_features,
mlp, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='PPool',
bn_decay=bn_decay)
new_features = tf.reduce_max(input_tensor=new_grouped_features,
axis=[2]) # max pooling
return new_xyz, new_features
else: # EnhancedPointConv
# new_xyz: B N 3, grouped_features: B N S C+3
# conv_phi function: grouped version
new_grouped_features = tf_util.conv2d_group(
grouped_features,
mlp,
kernel_size=[1, 1],
group_num=group_num,
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='PhiConv',
bn_decay=bn_decay)
new_features = tf.reduce_max(input_tensor=new_grouped_features,
axis=[2]) # max pooling
# conv_psi
new_features = tf_util.conv1d(new_features,
mlp // 4,
kernel_size=1,
padding='VALID',
stride=1,
bn=bn,
is_training=is_training,
scope='PsiConv',
bn_decay=bn_decay)
# features: B N 1 Cin, B N 1 Cout/4 => B N 1 Cin+Cout/4
all_new_features = tf.concat([features, new_features], axis=-1)
# batch normalize the concatenate output
all_new_features = tf_util.batch_norm_for_conv2d(
all_new_features,
is_training=is_training,
scope='BNConcat',
bn_decay=bn_decay)
all_new_features = tf.nn.relu(all_new_features)
return new_xyz, all_new_features
# GlobalPooling
else:
grouped_features = group_all(xyz, features)
grouped_features = tf_util.conv2d(grouped_features,
mlp, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='globalpooling',
bn_decay=bn_decay)
new_features = tf.reduce_max(input_tensor=grouped_features,
axis=[2]) # max pooling
return new_features
def pct_model(point_cloud, is_training, bn_decay=None):
"""
:param point_cloud:
:param is_training:
:param bn_decay:
:return:
"""
# point_cloud -> [batch_size, num_point, 3]
batch_size = point_cloud.get_shape()[0].value
point_dim = point_cloud.get_shape()[2].value
"""Input Embedding module"""
# [batch_size, num_point, 64]
x = tf_util.conv1d(point_cloud, 64, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope='conv0', bn_decay=bn_decay)
x = tf_util.conv1d(x, 64, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
""" Sample and Group """
new_xyz, new_feature, _, _ = pointnet_util.sample_and_group(
npoint=512, radius=0.15,
nsample=32, xyz=point_cloud,
points=x, knn=True, use_xyz=True)
# print(new_xyz.shape)
# print("new_feature.shape", new_feature.shape)
feature_0 = local_op(new_feature, out_dim=128, scope="SG1",
bn_decay=bn_decay, is_training=is_training)
new_xyz, new_feature, _, _ = pointnet_util.sample_and_group(
npoint=256, radius=0.2,
nsample=32, xyz=new_xyz,
points=feature_0,
knn=True, use_xyz=True)
# NHC
feature_1 = local_op(new_feature, out_dim=256, scope="SG2",
bn_decay=bn_decay, is_training=is_training)
#
# NHC
x = pt_last(feature_1, scope="pct_layer", out_dim=256,
bn_decay=bn_decay, is_training=is_training)
# concat in C (NHC) axis
x = tf.concat([x, feature_1], axis=-1)
x = tf_util.conv1d(x, 1024, kernel_size=1,
padding='VALID', stride=1, bn=True,
is_biases=False, is_training=is_training,
scope='conv2', bn_decay=bn_decay, activation_fn=None)
x = tf.nn.leaky_relu(x, alpha=0.2)
x = tf.reduce_max(x, axis=1)
"""
++++++++++++++++++++++++++++++++++++++++
Decoder
++++++++++++++++++++++++++++++++++++++++
"""
x = tf_util.fully_connected(x, 512, bn=True, is_training=is_training, is_biases=False,
scope='fc1', bn_decay=bn_decay)
x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp1')
x = tf_util.fully_connected(x, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp2')
x = tf_util.fully_connected(x, 40, activation_fn=None, scope='fc3')
return x
def pointnet_sa_module_msg(xyz, points, radius_list, nsample_list,
mlp_list, is_training, bn_decay, bn,
fps_sample_range_list, fps_method_list, npoint_list,
former_fps_idx, use_attention, scope,
dilated_group, vote_ctr=None, aggregation_channel=None,
debugging=False,
epsilon=1e-5, img_features=None):
''' PointNet Set Abstraction (SA) module with Multi-Scale Grouping (MSG)
Input:
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor
npoint: int -- points sampled in farthest point sampling
radius_list: list of float32 -- search radius in local region
nsample_list: list of int32 -- how many points in each local region
mlp_list: list of list of int32 -- output size for MLP on each point
fps_method: 'F-FPS', 'D-FPS', 'FS'
fps_start_idx:
Return:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, \sum_k{mlp[k][-1]}) TF tensor
'''
bs = xyz.get_shape().as_list()[0]
with tf.variable_scope(scope) as sc:
cur_fps_idx_list = []
last_fps_end_index = 0
for fps_sample_range, fps_method, npoint in zip(fps_sample_range_list, fps_method_list, npoint_list):
tmp_xyz = tf.slice(xyz, [0, last_fps_end_index, 0], [-1, fps_sample_range, -1])
tmp_points = tf.slice(points, [0, last_fps_end_index, 0], [-1, fps_sample_range, -1])
if npoint == 0:
last_fps_end_index += fps_sample_range
continue
if vote_ctr is not None:
npoint = vote_ctr.get_shape().as_list()[1]
fps_idx = tf.tile(tf.reshape(tf.range(npoint), [1, npoint]), [bs, 1])
elif fps_method == 'FS':
features_for_fps = tf.concat([tmp_xyz, tmp_points], axis=-1)
features_for_fps_distance = model_util.calc_square_dist(features_for_fps, features_for_fps, norm=False)
fps_idx_1 = farthest_point_sample_with_distance(npoint, features_for_fps_distance)
fps_idx_2 = farthest_point_sample(npoint, tmp_xyz)
fps_idx = tf.concat([fps_idx_1, fps_idx_2], axis=-1) # [bs, npoint * 2]
elif npoint == tmp_xyz.get_shape().as_list()[1]:
fps_idx = tf.tile(tf.reshape(tf.range(npoint), [1, npoint]), [bs, 1])
elif fps_method == 'F-FPS':
features_for_fps = tf.concat([tmp_xyz, tmp_points], axis=-1)
features_for_fps_distance = model_util.calc_square_dist(features_for_fps, features_for_fps, norm=False)
fps_idx = farthest_point_sample_with_distance(npoint, features_for_fps_distance)
else: # D-FPS
fps_idx = farthest_point_sample(npoint, tmp_xyz)
fps_idx = fps_idx + last_fps_end_index
cur_fps_idx_list.append(fps_idx)
last_fps_end_index += fps_sample_range
fps_idx = tf.concat(cur_fps_idx_list, axis=-1)
if former_fps_idx is not None:
fps_idx = tf.concat([fps_idx, former_fps_idx], axis=-1)
if vote_ctr is not None:
new_xyz = gather_point(vote_ctr, fps_idx)
else:
new_xyz = gather_point(xyz, fps_idx)
# if deformed_xyz is not None, then no attention model
if use_attention:
# first gather the points out
new_points = gather_point(points, fps_idx) # [bs, npoint, c]
# choose farthest feature to center points
# [bs, npoint, ndataset]
relation = model_util.calc_square_dist(new_points, points)
# choose these points with largest distance to center_points
_, relation_idx = tf.nn.top_k(relation, k=relation.shape.as_list()[-1])
idx_list, pts_cnt_list = [], []
cur_radius_list = []
for i in range(len(radius_list)):
radius = radius_list[i]
nsample = nsample_list[i]
if dilated_group:
# cfg.POINTNET.DILATED_GROUPING
if i == 0: min_radius = 0.
else: min_radius = radius_list[i - 1]
idx, pts_cnt = query_ball_point_dilated(min_radius, radius, nsample, xyz, new_xyz)
elif use_attention:
idx, pts_cnt = query_ball_point_withidx(radius, nsample, xyz, new_xyz, relation_idx)
else:
idx, pts_cnt = query_ball_point(radius, nsample, xyz, new_xyz)
idx_list.append(idx)
pts_cnt_list.append(pts_cnt)
# debugging
debugging_list = []
new_points_list = []
for i in range(len(radius_list)):
nsample = nsample_list[i]
idx, pts_cnt = idx_list[i], pts_cnt_list[i]
radius = radius_list[i]
pts_cnt_mask = tf.cast(tf.greater(pts_cnt, 0), tf.int32) # [bs, npoint]
pts_cnt_fmask = tf.cast(pts_cnt_mask, tf.float32)
idx = idx * tf.expand_dims(pts_cnt_mask, axis=2) # [bs, npoint, nsample]
grouped_xyz = group_point(xyz, idx)
original_xyz = grouped_xyz
grouped_xyz -= tf.expand_dims(new_xyz, 2)
grouped_points = group_point(points, idx)
grouped_points = tf.concat([grouped_points, grouped_xyz], axis=-1)
for j, num_out_channel in enumerate(mlp_list[i]):
grouped_points = tf_util.conv2d(grouped_points,
num_out_channel,
[1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv%d_%d' % (i, j),
bn_decay=bn_decay)
new_points = tf.reduce_max(grouped_points, axis=[2])
new_points *= tf.expand_dims(pts_cnt_fmask, axis=-1)
new_points_list.append(new_points)
if len(new_points_list) > 0:
new_points_concat = tf.concat(new_points_list, axis=-1)
if img_features is not None:
new_points_concat = tf.concat([new_points_concat, img_features], axis=-1)
if cfg.MODEL.NETWORK.AGGREGATION_SA_FEATURE:
new_points_concat = tf_util.conv1d(new_points_concat, aggregation_channel, 1, padding='VALID', bn=bn, is_training=is_training, scope='ensemble', bn_decay=bn_decay)
else:
new_points_concat = gather_point(points, fps_idx)
return new_xyz, new_points_concat, fps_idx
def get_model(point_cloud, is_training, num_class, hyperparams, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
if hyperparams['use_color'] or hyperparams['use_z_feature']:
feature_size = 3 * int(hyperparams['use_color']) + int(
hyperparams['use_z_feature'])
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, feature_size])
else:
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(
l0_xyz,
l0_points,
npoint=hyperparams['l1_npoint'],
radius=hyperparams['l1_radius'],
nsample=hyperparams['l1_nsample'],
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(
l1_xyz,
l1_points,
npoint=hyperparams['l2_npoint'],
radius=hyperparams['l2_radius'],
nsample=hyperparams['l2_nsample'],
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(
l2_xyz,
l2_points,
npoint=hyperparams['l3_npoint'],
radius=hyperparams['l3_radius'],
nsample=hyperparams['l3_nsample'],
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(
l3_xyz,
l3_points,
npoint=hyperparams['l4_npoint'],
radius=hyperparams['l4_radius'],
nsample=hyperparams['l4_nsample'],
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet++, input is BxNxF, output Bxnum_class """
end_points = {}
# COG
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
# Features
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, -1])
end_points['l0_xyz'] = l0_xyz
# Set Abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz,
l0_points,
npoint=4096,
radius_list=[0.5, 1, 2],
nsample_list=[8, 16, 32],
mlp_list=[[32, 32, 64],
[64, 64, 128],
[128, 128, 256]],
is_training=is_training,
pooling='max_and_avg',
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz,
l1_points,
npoint=256,
radius_list=[2, 4, 8],
nsample_list=[32, 64, 128],
mlp_list=[[128, 128, 256],
[256, 256, 512],
[256, 256, 512]],
is_training=is_training,
pooling='max_and_avg',
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points = pointnet_sa_module_msg(l2_xyz,
l2_points,
npoint=32,
radius_list=[4, 8, 16],
nsample_list=[64, 64, 128],
mlp_list=[[256, 256, 512],
[512, 512, 1024],
[512, 1024, 1024]],
is_training=is_training,
pooling='max_and_avg',
bn_decay=bn_decay,
scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [512, 512],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
