def extractor_layers(point_cloud,
n,
is_training,
bn_decay,
bn=True,
n_out=32,
scope='extractor'):
input_image = tf.expand_dims(point_cloud, 2)
for i in range(n):
net, conv_ker = tf_util.inception(input_image,
n_out,
scope=f'seq_conv{i}',
kernel_heights=[1, 3, 5, 7],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 2, 2, 2],
return_kernel=True,
bn=bn,
bn_decay=bn_decay,
is_training=is_training)
if i == 0:
ker = conv_ker
return tf.squeeze(net), ker
def get_model(point_cloud,
is_training,
n_classes,
bn_decay=None,
weight_decay=None,
inception=True,
**kwargs):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
if point_cloud.shape[2] > 3:
l0_xyz = point_cloud[:, :, :3]
l0_points = point_cloud[:, :, 3:]
else:
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
input_image = tf.expand_dims(point_cloud, 2)
net, conv_ker = tf_util.inception(input_image,
64,
scope='seq_conv1',
kernel_heights=[1, 3, 5, 7],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 2, 2, 2],
return_kernel=True,
bn=True,
bn_decay=bn_decay,
is_training=is_training)
end_points['conv_ker'] = conv_ker
net, conv_ker = tf_util.inception(input_image,
32,
scope='seq_conv2',
kernel_heights=[1, 3, 5, 7],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 2, 2, 2],
return_kernel=True,
bn=True,
bn_decay=bn_decay,
is_training=is_training)
conv_net = tf.squeeze(net)
l0_points = tf.concat([conv_net, l0_points], axis=-1)
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1',
use_nchw=True,
weight_decay=weight_decay)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2',
weight_decay=weight_decay)
l3_xyz, l3_points, _, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3',
weight_decay=weight_decay)
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay,
weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay,
weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net,
n_classes,
activation_fn=None,
scope='fc3')
return net, end_points
def pointnet_sa_module(xyz,
points,
npoint,
radius,
nsample,
mlp,
mlp2,
group_all,
is_training,
bn_decay,
scope,
inception=False,
weight_decay=None,
bn=True,
pooling='max',
knn=False,
use_xyz=True,
use_nchw=False,
sorted_sampling=True):
''' PointNet Set Abstraction (SA) Module
Input:
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor
npoint: int32 -- #points sampled in farthest point sampling
radius: float32 -- search radius in local region
nsample: int32 -- how many points in each local region
mlp: list of int32 -- output size for MLP on each point
mlp2: list of int32 -- output size for MLP on each region
group_all: bool -- group all points into one PC if set true, OVERRIDE
npoint, radius and nsample settings
use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features
use_nchw: bool, if True, use NCHW data format for conv2d, which is usually faster than NHWC format
Return:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor
idx: (batch_size, npoint, nsample) int32 -- indices for local regions
'''
data_format = 'NCHW' if use_nchw else 'NHWC'
with tf.variable_scope(scope) as sc:
# Sample and Grouping
if group_all:
nsample = xyz.get_shape()[1].value
new_xyz, new_points, idx, grouped_xyz = sample_and_group_all(
xyz,
points,
use_xyz,
)
else:
new_xyz, new_points, idx, grouped_xyz = sample_and_group(
npoint,
radius,
nsample,
xyz,
points,
knn,
use_xyz,
sorted_sampling=sorted_sampling)
# Point Feature Embedding
if use_nchw: new_points = tf.transpose(new_points, [0, 3, 1, 2])
kernel = None
if inception:
for i, num_out_channel in enumerate(mlp):
new_points, ker = tf_util.inception(
new_points,
num_out_channel,
kernels_fraction=[3, 2, 2, 1],
padding='SAME',
bn=bn,
is_training=is_training,
scope='conv%d' % (i),
bn_decay=bn_decay,
weight_decay=weight_decay,
return_kernel=True,
data_format=data_format)
if i == 0:
kernel = ker
else:
for i, num_out_channel in enumerate(mlp):
new_points, ker = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
return_kernel=True,
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv%d' % (i),
bn_decay=bn_decay,
weight_decay=weight_decay,
data_format=data_format)
if i == 0:
kernel = ker
if use_nchw: new_points = tf.transpose(new_points, [0, 2, 3, 1])
# Pooling in Local Regions
if pooling == 'max':
new_points = tf.reduce_max(new_points,
axis=[2],
keepdims=True,
name='maxpool')
elif pooling == 'avg':
new_points = tf.reduce_mean(new_points,
axis=[2],
keepdims=True,
name='avgpool')
elif pooling == 'weighted_avg':
with tf.variable_scope('weighted_avg'):
dists = tf.norm(grouped_xyz, axis=-1, ord=2, keepdims=True)
exp_dists = tf.exp(-dists * 5)
weights = exp_dists / tf.reduce_sum(
exp_dists, axis=2,
keepdims=True) # (batch_size, npoint, nsample, 1)
new_points *= weights # (batch_size, npoint, nsample, mlp[-1])
new_points = tf.reduce_sum(new_points, axis=2, keepdims=True)
elif pooling == 'max_and_avg':
max_points = tf.reduce_max(new_points,
axis=[2],
keepdims=True,
name='maxpool')
avg_points = tf.reduce_mean(new_points,
axis=[2],
keepdims=True,
name='avgpool')
new_points = tf.concat([avg_points, max_points], axis=-1)
# [Optional] Further Processing
if mlp2 is not None:
if use_nchw: new_points = tf.transpose(new_points, [0, 3, 1, 2])
for i, num_out_channel in enumerate(mlp2):
new_points = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv_post_%d' % (i),
bn_decay=bn_decay,
weight_decay=weight_decay,
data_format=data_format)
if use_nchw: new_points = tf.transpose(new_points, [0, 2, 3, 1])
new_points = tf.squeeze(new_points,
[2]) # (batch_size, npoints, mlp2[-1])
return new_xyz, new_points, idx, kernel
def conv_network(point_cloud,
is_training,
scope,
channels_out=1024,
bn_decay=None):
point_cloud = tf.expand_dims(point_cloud, 2)
net, kernel = tf_util.inception(point_cloud,
64,
scope=f'{scope}_1',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training,
return_kernel=True)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_1_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
128,
scope=f'{scope}_2',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_2_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
128,
scope=f'{scope}_3',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_3_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
256,
scope=f'{scope}_4',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_4_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
512,
scope=f'{scope}_5',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_5_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
channels_out,
scope=f'{scope}_6',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.avg_pool2d(net,
kernel_size=[net.shape[1], 1],
scope=f'{scope}_GAP',
stride=[net.shape[1], 1],
padding='SAME')
net = tf.squeeze(net)
return net, kernel
def get_model(point_cloud, is_training, n_classes, bn_decay=None, weight_decay=None, inception=True, **kwargs):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
channels_size = point_cloud.get_shape()[2].value
end_points = {}
if point_cloud.shape[2] > 3:
l0_xyz = point_cloud[:, :, :3]
l0_points = point_cloud[:, :, 3:]
else:
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
input_image = tf.expand_dims(point_cloud, 2)
net, conv_ker = tf_util.inception(input_image, 64, scope='seq_conv1',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 2, 2, 2],
return_kernel=True,
bn=True, bn_decay=bn_decay,
is_training=is_training)
end_points['conv_ker'] = conv_ker
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_1',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 64, scope='seq_conv2',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_2',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 128, scope='seq_conv3',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_3',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 256, scope='seq_conv4',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_4',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 512, scope='seq_conv5',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_5',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 1024, scope='seq_conv6',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.avg_pool2d(net, kernel_size=[net.shape[1], 1], scope='seq_global_avg_pool', stride=[net.shape[1],1], padding='SAME')
# # Symmetric function: max pooling
# net = tf_util.max_pool2d(net, [num_point,1],
# padding='VALID', scope='maxpool')
conv_net = tf.squeeze(net)
# Set abstraction layers
# Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4).
# So we only use NCHW for layer 1 until this issue can be resolved.
l1_xyz, l1_points, l1_indices, pt_ker = pointnet_sa_module(l0_xyz, l0_points, npoint=256, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True)
end_points['pt_ker'] = pt_ker
l2_xyz, l2_points, l2_indices, _ = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=128, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
net = tf.reshape(l3_points, [batch_size, -1])
net = tf.concat([conv_net, net], axis=-1)
end_points['feature_vector'] = net
# Fully connected layers
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1') # my change
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay, weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net, n_classes, activation_fn=None, scope='fc3')
return net, end_points