def pointnet_sa_module(xyz, points, npoint, radius, nsample, mlp, mlp2, group_all, is_training, bn_decay, scope, bn=True, pooling='max', tnet_spec=None, knn=False, use_xyz=True):
''' PointNet Set Abstraction (SA) Module
Input:
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor
npoint: int32 -- #points sampled in farthest point sampling
radius: float32 -- search radius in local region
nsample: int32 -- how many points in each local region
mlp: list of int32 -- output size for MLP on each point
mlp2: list of int32 -- output size for MLP on each region
group_all: bool -- group all points into one PC if set true, OVERRIDE
npoint, radius and nsample settings
use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features
Return:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor
idx: (batch_size, npoint, nsample) int32 -- indices for local regions
'''
with tf.variable_scope(scope) as sc:
if group_all:
nsample = xyz.get_shape()[1].value
# (batch_size, 1, ndataset, 3+channel)
new_xyz, new_points, idx, grouped_xyz = sample_and_group_all(xyz, points, use_xyz)
else:
# new_points:(batch_size, npoint, nsample, 3+channel) TF tensor
new_xyz, new_points, idx, grouped_xyz = sample_and_group(npoint, radius, nsample, xyz, points, tnet_spec, knn, use_xyz)
for i, num_out_channel in enumerate(mlp): #index and numerate # 一个mlp（多层感知机）
new_points = tf_util.conv2d(new_points, num_out_channel, [1,1],
padding='VALID', stride=[1,1],
bn=bn, is_training=is_training,
scope='conv%d'%(i), bn_decay=bn_decay)
### pooling 对采样维进行降维
if pooling=='avg':
new_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1')
elif pooling=='weighted_avg':
with tf.variable_scope('weighted_avg1'):
dists = tf.norm(grouped_xyz,axis=-1,ord=2,keep_dims=True)
exp_dists = tf.exp(-dists * 5)
weights = exp_dists/tf.reduce_sum(exp_dists,axis=2,keep_dims=True) # (batch_size, npoint, nsample, 1)
new_points *= weights # (batch_size, npoint, nsample, mlp[-1])
new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True)
elif pooling=='max':
#参数2--reduction_indices:在哪一维上求解 如果keep_dims为true，则减小的维度将保留为长度1
new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True) #
elif pooling=='min':
new_points = tf_util.max_pool2d(-1*new_points, [1,nsample], stride=[1,1], padding='VALID', scope='minpool1')
elif pooling=='max_and_avg':
avg_points = tf_util.max_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='maxpool1')
max_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1')
new_points = tf.concat([avg_points, max_points], axis=-1)
if mlp2 is None: mlp2 = []
for i, num_out_channel in enumerate(mlp2):
new_points = tf_util.conv2d(new_points, num_out_channel, [1,1],
padding='VALID', stride=[1,1],
bn=bn, is_training=is_training,
scope='conv_post_%d'%(i), bn_decay=bn_decay)
new_points = tf.squeeze(new_points, [2]) # 压缩数目为1的维度 (batch_size, npoints, mlp2[-1])
return new_xyz, new_points, idx # mew_xyz:中心点 new_points:中心点采样后计算得到的featrue
def pointnet_sa_module(xyz, points, npoint, radius, nsample, mlp, mlp2, group_all, is_training, bn_decay, scope, bn=True, pooling='max', sort_k=None, tnet_spec=None, knn=False, use_xyz=True):
''' PointNet Set Abstraction (SA) Module
Input:
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor
npoint: int32 -- #points sampled in farthest point sampling
radius: float32 -- search radius in local region
nsample: int32 -- how many points in each local region
mlp: list of int32 -- output size for MLP on each point
mlp2: list of int32 -- output size for MLP on each region
group_all: bool -- group all points into one PC if set true, OVERRIDE
npoint, radius and nsample settings
use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features
Return:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor
idx: (batch_size, npoint, nsample) int32 -- indices for local regions
'''
with tf.variable_scope(scope) as sc:
if group_all:
nsample = xyz.get_shape()[1].value
new_xyz, new_points, idx, grouped_xyz = sample_and_group_all(xyz, points, use_xyz)
else:
new_xyz, new_points, idx, grouped_xyz = sample_and_group(npoint, radius, nsample, xyz, points, tnet_spec, knn, use_xyz)
for i, num_out_channel in enumerate(mlp):
new_points = tf_util.conv2d(new_points, num_out_channel, [1,1],
padding='VALID', stride=[1,1],
bn=bn, is_training=is_training,
scope='conv%d'%(i), bn_decay=bn_decay)
if pooling=='avg':
new_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1')
elif pooling=='weighted_avg':
with tf.variable_scope('weighted_avg1'):
dists = tf.norm(grouped_xyz,axis=-1,ord=2,keep_dims=True)
exp_dists = tf.exp(-dists * 5)
weights = exp_dists/tf.reduce_sum(exp_dists,axis=2,keep_dims=True) # (batch_size, npoint, nsample, 1)
new_points *= weights # (batch_size, npoint, nsample, mlp[-1])
new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True)
elif pooling=='max':
new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True)
elif pooling=='min':
new_points = tf_util.max_pool2d(-1*new_points, [1,nsample], stride=[1,1], padding='VALID', scope='minpool1')
elif pooling=='max_and_avg':
max_points = tf.reduce_max(new_points, axis=[2], keep_dims=True, name='maxpool1')
avg_points = tf.reduce_mean(new_points, axis=[2], keep_dims=True, name='avgpool1')
new_points = tf.concat([avg_points, max_points], axis=-1)
elif pooling=='sort':
new_points = tf.transpose(tf.nn.top_k((tf.transpose(new_points, perm=[0, 1, 3, 2])), sort_k)[0], perm=[0, 1, 3, 2]) # (?, 2048, 1, 64)
print new_points.get_shape()
if mlp2 is None: mlp2 = []
for i, num_out_channel in enumerate(mlp2):
new_points = tf_util.conv2d(new_points, num_out_channel, [1,1],
padding='VALID', stride=[1,1],
bn=bn, is_training=is_training,
scope='conv_post_%d'%(i), bn_decay=bn_decay)
if pooling != 'sort':
new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1])
print new_points.get_shape()
return new_xyz, new_points, idx
def feature_transform_net(inputs, is_training, bn_decay=None, K=64):
""" Feature Transform Net, input is BxNx1xK
Return:
Transformation matrix of size KxK """
batch_size = inputs.get_shape()[0].value
num_point = inputs.get_shape()[1].value
net = tf_util.conv2d(inputs,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='tconv1',
bn_decay=bn_decay)
#net = tf_util.conv2d(net, 128, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='tconv2', bn_decay=bn_decay)
#net = tf_util.conv2d(net, 1024, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='tconv3', bn_decay=bn_decay)
maxpool_net = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='ttmaxpool')
avg_pool2d_net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='ttavgpool')
net = tf.concat([avg_pool2d_net, maxpool_net], axis=3)
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='tfc1',
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='tfc2',
bn_decay=bn_decay)
with tf.variable_scope('transform_feat') as sc:
weights = tf.get_variable('weights', [256, K * K],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases = tf.get_variable('biases', [K * K],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32)
transform = tf.matmul(net, weights)
transform = tf.nn.bias_add(transform, biases)
transform = tf.reshape(transform, [batch_size, K, K])
return transform
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud, is_training, bn_decay, K=6)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image, 64, [1,6],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
print("net = ", net.shape)
with tf.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = tf_util.conv2d(net_transformed, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
# Symmetric function: max pooling
max_net = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
avg_net = tf_util.avg_pool2d(net, [num_point,1],
padding='VALID', scope='avgpool')
max_avg_net = tf.concat([max_net, avg_net], 3)
net = tf.reshape(max_avg_net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
input_image = tf.expand_dims(point_cloud, -1)
# Point functions (MLP implemented as conv2d)
net = tf_util.conv2d(input_image, 32, [1,9],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
#net = tf_util.conv2d(net, 128, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='conv2', bn_decay=bn_decay)
#net = tf_util.conv2d(net, 128, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='conv3', bn_decay=bn_decay)
#net = tf.concat([net,net1, net2],axis=-1)
#net = tf_util.conv2d(net, 128, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='conv4', bn_decay=bn_decay)
#net = tf_util.conv2d(net, 1024, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='conv5', bn_decay=bn_decay)
# Symmetric function: max pooling
maxpool_net = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
avg_pool2d_net = tf_util.avg_pool2d(net, [num_point,1],
padding='VALID', scope='avgpool')
net = tf.concat([avg_pool2d_net, maxpool_net], axis=3)
#net = tf.concat([tf.reduce_sum(net,axis=1,keep_dims=True), net1], axis=3)
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def pointnet_avg(source_point_cloud, template_point_cloud, is_training, bn_decay=None,out_features=1024):
point_cloud = tf.concat([source_point_cloud, template_point_cloud], 0)
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
input_image = tf.expand_dims(point_cloud, -1)
net = tf_util.conv2d(input_image, 64, [1, 3],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1, 1],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1, 1],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, out_features, [1, 1],
padding='VALID', stride=[1, 1],
bn=False, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID', scope='maxpool')
net = tf.reshape(net, [batch_size, -1])
source_global_feature = tf.slice(net, [0, 0], [int(batch_size / 2), out_features])
template_global_feature = tf.slice(net, [int(batch_size / 2), 0], [int(batch_size / 2), out_features])
return source_global_feature, template_global_feature
def input_transform_net(point_cloud, is_training, bn_decay=None, K=9):
""" Input (XYZ) Transform Net, input is BxNx3 gray image
Return:
Transformation matrix of size 3xK """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -1)
net = tf_util.conv2d(input_image,
128, [1, 9],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='tconv1',
bn_decay=bn_decay)
#net = tf_util.conv2d(net, 128, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='tconv2', bn_decay=bn_decay)
#net = tf_util.conv2d(net, 1024, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='tconv3', bn_decay=bn_decay)
maxpool_net = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='tmaxpool')
avg_pool2d_net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='tavgpool')
net = tf.concat([avg_pool2d_net, maxpool_net], axis=3)
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='tfc1',
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='tfc2',
bn_decay=bn_decay)
with tf.variable_scope('transform_XYZ') as sc:
assert (K == 9)
weights = tf.get_variable('weights', [256, 9 * K],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases = tf.get_variable('biases', [9 * K],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32)
transform = tf.matmul(net, weights)
transform = tf.nn.bias_add(transform, biases)
transform = tf.reshape(transform, [batch_size, 9, K])
return transform
def get_model(point_cloud, is_training, bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -1)
# CONV
net = tf_util.conv2d(input_image,
64, [1, 12],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
points_feat1 = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# MAX and AVG
pc_feat1_max = tf_util.max_pool2d(points_feat1, [num_point, 1],
padding='VALID',
scope='maxpool1')
pc_feat1_avg = tf_util.avg_pool2d(points_feat1, [num_point, 1],
padding='VALID',
scope='avgpool1')
pc_feat1 = tf.concat([pc_feat1_max, pc_feat1_avg], 3)
# FC
pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1])
pc_feat1 = tf_util.fully_connected(pc_feat1,
256,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
pc_feat1 = tf_util.fully_connected(pc_feat1,
128,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
print(pc_feat1)
# CONCAT
pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]),
[1, num_point, 1, 1])
points_feat1_concat = tf.concat(axis=3,
values=[points_feat1, pc_feat1_expand])
# CONV
net = tf_util.conv2d(points_feat1_concat,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv6')
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv7')
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.conv2d(net,
13, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
scope='conv8')
net = tf.squeeze(net, [2])
return net
def conv_network(point_cloud,
is_training,
scope,
channels_out=1024,
bn_decay=None):
point_cloud = tf.expand_dims(point_cloud, 2)
net, kernel = tf_util.inception(point_cloud,
64,
scope=f'{scope}_1',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training,
return_kernel=True)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_1_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
128,
scope=f'{scope}_2',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_2_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
128,
scope=f'{scope}_3',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_3_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
256,
scope=f'{scope}_4',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_4_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
512,
scope=f'{scope}_5',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net,
kernel_size=[2, 1],
scope=f'{scope}_5_max_pool',
stride=[2, 1],
padding='SAME')
net = tf_util.inception(net,
channels_out,
scope=f'{scope}_6',
kernel_heights=[1, 3, 5, 7],
dilations=[3, 1],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 3, 2, 1],
bn=True,
bn_decay=bn_decay,
is_training=is_training)
net = tf_util.avg_pool2d(net,
kernel_size=[net.shape[1], 1],
scope=f'{scope}_GAP',
stride=[net.shape[1], 1],
padding='SAME')
net = tf.squeeze(net)
return net, kernel
def get_instance_seg_v1_net(point_cloud, one_hot_vec, is_training, bn_decay,
end_points):
''' 3D instance segmentation PointNet v1 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
net = tf.expand_dims(point_cloud, 2)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
point_feat = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(point_feat,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
#global_feat = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool')
global_feat = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='avgpool')
global_feat = tf.concat(
[global_feat,
tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)],
axis=3)
global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])
concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand])
net = tf_util.conv2d(concat_feat,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv6',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv7',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv8',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv9',
bn_decay=bn_decay)
net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5)
logits = tf_util.conv2d(net,
2, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
scope='conv10')
logits = tf.squeeze(logits, [2]) # BxNx2
return logits, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \
batch_size, num_point, weight_decay, bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
end_points = {}
with tf.variable_scope('transform_net1') as sc:
K = 6
transform = get_transform(point_cloud, is_training, bn_decay, K)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
out1 = tf_util.conv2d(input_image, 64, [1,6], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay)
out2 = tf_util.conv2d(out1, 128, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay)
out3 = tf_util.conv2d(out2, 128, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
K = 128
transform = get_transform_K(out3, is_training, bn_decay, K)
end_points['transform'] = transform
squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128])
net_transformed = tf.matmul(squeezed_out3, transform)
net_transformed = tf.expand_dims(net_transformed, [2])
out4 = tf_util.conv2d(net_transformed, 512, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay)
out5 = tf_util.conv2d(out4, 2048, [1,1], padding='VALID', stride=[1,1],
bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay)
out_max = tf_util.max_pool2d(out5, [num_point,1], padding='VALID', scope='maxpool')
out_avg = tf_util.avg_pool2d(out5, [num_point,1], padding='VALID', scope='avgpool')
out_max = tf.concat(axis=3, values=[out_max, out_avg])
# classification network
net = tf.reshape(out_max, [batch_size, -1])
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1')
net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3')
# segmentation network
one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num])
out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand])
expand = tf.tile(out_max, [1, num_point, 1, 1])
concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5])
net2 = tf_util.conv2d(concat, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay,
bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay)
net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1')
net2 = tf_util.conv2d(net2, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay,
bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay)
net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2')
net2 = tf_util.conv2d(net2, 128, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay,
bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay)
net2 = tf_util.conv2d(net2, part_num, [1,1], padding='VALID', stride=[1,1], activation_fn=None,
bn=False, scope='seg/conv4', weight_decay=weight_decay)
net2 = tf.reshape(net2, [batch_size, num_point, part_num])
return net, net2, end_points
def vgg16(input, is_training, output_dim, bn_decay=None):
"""
objective: build the network
param:
input: image patch in brg (after pre-processing), [batch, height, width, 3],
is_training: training or testing
bn_decay: None
output_dim: final feature dimension
"""
# block 1 (128x128x3 -> 64x64x64)
conv1_1 = tf_util.conv2d(input,
64, [3, 3],
scope='conv1_1',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
conv1_2 = tf_util.conv2d(conv1_1,
64, [3, 3],
scope='conv1_2',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
pool1 = tf_util.max_pool2d(conv1_2, [2, 2],
scope='pool1',
stride=[2, 2],
padding='VALID')
# block 2 (64x64x64 -> 32x32x128)
conv2_1 = tf_util.conv2d(pool1,
128, [3, 3],
scope='conv2_1',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
conv2_2 = tf_util.conv2d(conv2_1,
128, [3, 3],
scope='conv2_2',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
pool2 = tf_util.max_pool2d(conv2_2, [2, 2],
scope='pool2',
stride=[2, 2],
padding='VALID')
# block 3 (32x32x128 -> 16x16x256)
conv3_1 = tf_util.conv2d(pool2,
256, [3, 3],
scope='conv3_1',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
conv3_2 = tf_util.conv2d(conv3_1,
256, [3, 3],
scope='conv3_2',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
conv3_3 = tf_util.conv2d(conv3_2,
256, [3, 3],
scope='conv3_3',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
pool3 = tf_util.max_pool2d(conv3_3, [2, 2],
scope='pool3',
stride=[2, 2],
padding='VALID')
# block 4 (16x16x256 -> 16x16x512 -> 1x1x512)
conv4_1 = tf_util.conv2d(pool3,
512, [3, 3],
scope='conv4_1',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
conv4_2 = tf_util.conv2d(conv4_1,
512, [3, 3],
scope='conv4_2',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
conv4_3 = tf_util.conv2d(conv4_2,
512, [3, 3],
scope='conv4_3',
stride=[1, 1],
padding='SAME',
is_training=is_training,
bn_decay=bn_decay)
# net = tf_util.max_pool2d(net, [2,2], scope='pool4',stride=[2,2],padding='VALID')
pool4 = tf_util.avg_pool2d(conv4_3, [16, 16],
scope='pool4',
stride=[16, 16],
padding='VALID')
shape = pool4.get_shape().as_list()
dim = 1
for d in shape[1:]:
dim *= d
pool4 = tf.reshape(pool4, [-1, dim])
print(pool4.shape)
# block 5 (16x16x512 -> 1x512 -> 1x128)
fc1 = tf_util.fully_connected(pool4,
512,
scope='fc1',
is_training=is_training)
# fc1 = tf_util.dropout(fc1, keep_prob=0.7, is_training=is_training,
# scope='dp1')
#fc2 = tf_util.fully_connected(fc1, output_dim, scope='fc2', is_training = is_training)
fc2 = tf_util.fully_connected(fc1,
output_dim,
activation_fn=None,
scope='fc2',
is_training=is_training)
out = tf.nn.l2_normalize(fc2, dim=1)
return out
def get_model(point_cloud, is_training, n_classes, bn_decay=None, weight_decay=None, inception=True, **kwargs):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
channels_size = point_cloud.get_shape()[2].value
end_points = {}
if point_cloud.shape[2] > 3:
l0_xyz = point_cloud[:, :, :3]
l0_points = point_cloud[:, :, 3:]
else:
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
input_image = tf.expand_dims(point_cloud, 2)
net, conv_ker = tf_util.inception(input_image, 64, scope='seq_conv1',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 2, 2, 2],
return_kernel=True,
bn=True, bn_decay=bn_decay,
is_training=is_training)
end_points['conv_ker'] = conv_ker
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_1',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 64, scope='seq_conv2',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_2',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 128, scope='seq_conv3',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_3',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 256, scope='seq_conv4',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_4',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 512, scope='seq_conv5',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_5',
stride=[2,1], padding='SAME')
net = tf_util.inception(net, 1024, scope='seq_conv6',
kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1],
kernels_fraction=[3, 2, 2, 1],
bn=True, bn_decay=bn_decay,
is_training=is_training)
net = tf_util.avg_pool2d(net, kernel_size=[net.shape[1], 1], scope='seq_global_avg_pool', stride=[net.shape[1],1], padding='SAME')
# # Symmetric function: max pooling
# net = tf_util.max_pool2d(net, [num_point,1],
# padding='VALID', scope='maxpool')
conv_net = tf.squeeze(net)
# Set abstraction layers
# Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4).
# So we only use NCHW for layer 1 until this issue can be resolved.
l1_xyz, l1_points, l1_indices, pt_ker = pointnet_sa_module(l0_xyz, l0_points, npoint=256, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True)
end_points['pt_ker'] = pt_ker
l2_xyz, l2_points, l2_indices, _ = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=128, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
net = tf.reshape(l3_points, [batch_size, -1])
net = tf.concat([conv_net, net], axis=-1)
end_points['feature_vector'] = net
# Fully connected layers
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1') # my change
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay, weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net, n_classes, activation_fn=None, scope='fc3')
return net, end_points
def ec_to_weights(ec, mask, neighbornum, C=3, bn_decay=None, is_training=None):
# ec B N K*C
batch_size = ec.get_shape()[0]
point_num = ec.get_shape()[1]
#K = ec.get_shape()[2]
#ec_feature = edge_unit_with_ec(ec,mask,'max', config.neighbor_num, 32, scope='ec_conv1', bn=True, is_training=is_training,bn_decay=bn_decay)
#ec_feature = tf.expand_dims(ec_feature, -1)
#ec = tf.reshape(ec,[batch_size,point_num,neighbornum, C]) # B N K C
with tf.variable_scope('edge_net_without_pooling') as sc:
net, kernel = edge_unit(ec,
mask,
'max',
config.neighbor_num,
32,
scope='conv1',
bn=True,
is_training=is_training,
bn_decay=bn_decay)
net = tf_util.conv2d(ec,
64, [1, C],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='ec_conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='ec_conv2',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='ec_conv3',
bn_decay=bn_decay)
net = tf_util.avg_pool2d(net, [1, 1024], padding='VALID', scope='tmaxpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='tfc1',
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='tfc2',
bn_decay=bn_decay)
with tf.variable_scope('transform_XYZ') as sc:
if (K == 4):
weights = tf.get_variable('weights', [256, N * K],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases = tf.get_variable('biases', [4 * 4],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32)
transform = tf.matmul(net, weights)
transform = tf.nn.bias_add(transform, biases)
transform = tf.reshape(transform, [batch_size, 4, 4])
return transform
def get_model(point_cloud,
is_training,
num_class,
weight_decay=None,
bn_decay=None,
scname=''):
''' input: BxNxF
output:BxNx(cats*segms) '''
batch_size = point_cloud.get_shape()[0]
num_point = point_cloud.get_shape()[1]
num_feat = point_cloud.get_shape()[2]
k = 10
adj = tf_util.pairwise_distanceR(point_cloud[:, :, :3])
n_heads = 1
nn_idx = tf_util.knn(adj, k=k)
net, locals_transform, coefs = gap_block(k, n_heads, nn_idx, point_cloud,
point_cloud, ('filter0', 16),
bn_decay, weight_decay,
is_training, scname)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet01' + scname,
bn_decay=bn_decay)
net01 = net
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet02' + scname,
bn_decay=bn_decay)
net02 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=k)
adj_conv = nn_idx
n_heads = 1
net, locals_transform1, coefs2 = gap_block(k, n_heads, nn_idx, net,
point_cloud, ('filter1', 128),
bn_decay, weight_decay,
is_training, scname)
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet11' + scname,
bn_decay=bn_decay)
net11 = net
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet12' + scname,
bn_decay=bn_decay)
net12 = net
net = tf.concat(
[net01, net02, net11, net12, locals_transform, locals_transform1],
axis=-1)
net = tf_util.conv2d(net,
3, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='agg' + scname,
bn_decay=bn_decay)
net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='avgpool' + scname)
max_pool = net
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
activation_fn=tf.nn.relu,
scope='fc1' + scname,
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
128,
bn=True,
is_training=is_training,
activation_fn=tf.nn.relu,
scope='fc2' + scname,
bn_decay=bn_decay)
net = tf_util.fully_connected(net,
num_class,
activation_fn=None,
scope='fc3' + scname)
net = tf.squeeze(net)
return net, max_pool
def get_model(point_cloud, query_points, is_training, bn_decay=None):
""" range regression PointNet, input is BxNx3(point_cloud) and Bx2(query_points), output Bx1 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
#
query_points = tf.tile(
tf.expand_dims(query_points, [1]),
[1, num_point, 1]) # Now, query_points is with shape BxNx2
query_points = tf.concat(
[query_points, tf.zeros([batch_size, num_point, 1])],
2) # Now, query_points is with shape BxNx3
# point_cloud_ab = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 2]) # alpha and beta, shape:BxNx2
# point_cloud_range = tf.slice(point_cloud, [0, 0, 2], [-1, -1, 1]) # range, shape:BxNx1
# shift_ab = point_cloud_ab - query_points
shifted_pl = point_cloud - query_points
#
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(shifted_pl, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(shifted_pl, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=False,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=False,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = tf_util.conv2d(net_transformed,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=False,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=False,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=False,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# Symmetric function: average pooling
net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='avgpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=False,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=False,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 1, activation_fn=None, scope='fc3')
return net, end_points
def get_model_fine_tuning_test(point_cloud,
point_coords_in_voxels,
num_scale,
is_training,
bn_decay=None):
""" ConvNet baseline, input is BxNx3 gray image """
end_points = {}
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = tf_util.conv2d(net_transformed,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
points_feat1 = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# PYRAMID START #
# m x n x 1024
points_feat1 = tf.squeeze(points_feat1, [2])
print(points_feat1)
# m x n x (4 x 128 = 512)
points_feat1_concat = pyramid_nets.pyramid_convert_layer(
points_feat1,
point_coords_in_voxels,
num_scale, [256],
bn=True,
is_training=is_training,
bn_decay=bn_decay)
print(points_feat1_concat)
# m x n x 1 x 512
points_feat1_concat = tf.expand_dims(points_feat1_concat, [2])
# Concat pyramid global and local features
points_feat1 = tf.expand_dims(points_feat1, [2])
point_feat_concat = tf.concat(axis=3,
values=[points_feat1, points_feat1_concat])
# PYRAMID END #
# Symmetric function: max pooling
#net = tf_util.max_pool2d(point_feat_concat, [num_point,1], padding='VALID', scope='pyramid_maxpool')
net = tf_util.avg_pool2d(point_feat_concat, [num_point, 1],
padding='VALID',
scope='pyramid_maxpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='pyramid_fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='pyramid_dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='pyramid_fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='pyramid_dp2')
net = tf_util.fully_connected(net,
40,
activation_fn=None,
scope='pyramid_fc3')
return net, end_points
def get_model(
point_cloud,
is_training,
one_hot_labels,
bn_decay=None,
):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tfgan.features.condition_tensor_from_onehot(net, one_hot_labels)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = tf_util.conv2d(net_transformed,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
#net = tfgan.features.condition_tensor_from_onehot(net, one_hot_labels)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud,
is_training,
num_class,
global_pl,
params,
weight_decay=None,
bn_decay=None,
scname=''):
''' input: BxNxF
Use https://arxiv.org/pdf/1902.08570 as baseline
output:BxNx(cats*segms) '''
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
input_image = tf.expand_dims(point_cloud, -2)
k = params[0]
#adj = tf_util.pairwise_distance(point_cloud[:,:,:3])
adj = tf_util.pairwise_distanceR(point_cloud[:, :, :3])
n_heads = params[1]
nn_idx = tf_util.knn(adj, k=k)
net, locals_transform, coefs = gap_block(k, n_heads, nn_idx, point_cloud,
point_cloud,
('filter0', params[2]), bn_decay,
weight_decay, is_training, scname)
net = tf_util.conv2d(net,
params[3], [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='gapnet00',
bn_decay=bn_decay)
net00 = net
net = tf_util.conv2d(net,
params[4], [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet01' + scname,
bn_decay=bn_decay)
net01 = net
net = tf_util.conv2d(net,
params[5], [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet02' + scname,
bn_decay=bn_decay)
net02 = net
adj_matrix = tf_util.pairwise_distance(net)
nn_idx = tf_util.knn(adj_matrix, k=k)
adj_conv = nn_idx
n_heads = params[6]
net, locals_transform1, coefs2 = gap_block(k, n_heads, nn_idx, net,
point_cloud,
('filter1', params[7]),
bn_decay, weight_decay,
is_training, scname)
net = tf_util.conv2d(net,
params[8], [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='gapnet10',
bn_decay=bn_decay)
net10 = net
net = tf_util.conv2d(net,
params[9], [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet11' + scname,
bn_decay=bn_decay)
net11 = net
net = tf_util.conv2d(net,
params[10], [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='gapnet12' + scname,
bn_decay=bn_decay)
net12 = net
global_expand = tf.reshape(global_pl, [batch_size, 1, 1, -1])
global_expand = tf.tile(global_expand, [1, num_point, 1, 1])
global_expand = tf_util.conv2d(global_expand,
16, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='global_expand' + scname,
bn_decay=bn_decay)
net = tf.concat([
net00, net01, net02, net11, net12, global_expand, locals_transform,
locals_transform1
],
axis=-1)
net = tf_util.conv2d(net,
params[8], [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=tf.nn.relu,
bn=True,
is_training=is_training,
scope='agg' + scname,
bn_decay=bn_decay)
#net_tot = net
net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='avgpool' + scname)
expand = tf.tile(net, [1, num_point, 1, 1])
# net = tf.concat(axis=3, values=[expand,
# net_tot,
# ])
net = tf_util.conv2d(expand,
params[11], [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv2',
weight_decay=weight_decay,
is_dist=True)
net = tf_util.dropout(net,
keep_prob=0.6,
is_training=is_training,
scope='seg/dp1')
net = tf_util.conv2d(net,
params[11], [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv3',
weight_decay=weight_decay,
is_dist=True)
net = tf_util.dropout(net,
keep_prob=0.6,
is_training=is_training,
scope='seg/dp2')
net = tf_util.conv2d(net,
params[12], [1, 1],
padding='VALID',
stride=[1, 1],
bn_decay=bn_decay,
bn=True,
is_training=is_training,
scope='seg/conv4',
weight_decay=weight_decay,
is_dist=True)
net = tf_util.conv2d(net,
num_class, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
bn=False,
scope='seg/conv5',
weight_decay=weight_decay,
is_dist=True)
net = tf.cond(is_training, lambda: net, lambda: tf.nn.softmax(net))
net = tf.reshape(net, [batch_size, num_point, num_class])
return net, coefs, coefs2, adj_conv
def get_model_dss(point_cloud,
is_training,
is_outer,
bn_decay=None,
pool='max'):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
# with tf.variable_scope('transform_net1') as sc:
# transform = input_transform_net(point_cloud, is_training, bn_decay, K=3)
# point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud, -1)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
activation_fn=tf.nn.relu,
bn_decay=bn_decay)
side_1 = tf_util.conv2d(tf.reduce_max(net, axis=3, keepdims=True),
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='side_conv2',
activation_fn=None,
bn_decay=bn_decay)
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
activation_fn=None,
bn_decay=bn_decay)
net = tf.nn.relu(net + side_1)
side_2 = tf_util.conv2d(tf.reduce_max(net, axis=3, keepdims=True),
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='side_conv3',
activation_fn=None,
bn_decay=bn_decay)
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
activation_fn=None,
bn_decay=bn_decay)
net = tf.nn.relu(net + side_2)
tf.reduce_max(net, axis=3, keepdims=True)
# print(net.shape)
# with tf.variable_scope('transform_net2') as sc:
# transform = feature_transform_net(net, is_training, bn_decay, K=64)
# end_points['transform'] = transform
# net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
# net_transformed = tf.expand_dims(net_transformed, [2])
# net = tf_util.conv2d(net_transformed, 64, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='conv3', bn_decay=bn_decay)
# net = tf_util.conv2d(net, 128, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='conv4', activation_fn=tf.nn.relu, bn_decay=bn_decay)
# net = tf_util.conv2d(net, 1024, [1,1],
# padding='VALID', stride=[1,1],
# bn=True, is_training=is_training,
# scope='conv5', activation_fn=None, bn_decay=bn_decay)
# Symmetric function: max pooling
if pool == 'max':
net = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
elif pool == 'median':
net = tf_util.median_pool2d(net)
elif pool == 'median_inner':
net = tf.cond(
is_outer, lambda: tf_util.median_pool2d(net), lambda: tf_util.
sum_pool2d(net, [num_point, 1], padding='VALID', scope='sumpool_1')
+ tf.stop_gradient(
tf_util.median_pool2d(net) - tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_2')))
elif pool == 'median_outer':
net = tf.cond(
is_outer, lambda: tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf.
stop_gradient(
tf_util.median_pool2d(net) - tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_2')),
lambda: tf_util.median_pool2d(net))
elif pool == 'median_both':
net = tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID',
scope='sumpool_1') + tf.stop_gradient(
tf_util.median_pool2d(net) - tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_2'))
elif pool == 'avg':
net = tf_util.avg_pool2d(net, [num_point, 1],
padding='VALID',
scope='avgpool')
elif pool == 'inner':
net = tf.cond(
is_outer, lambda: tf_util.max_pool2d(
net, [num_point, 1], padding='VALID', scope='maxpool'),
lambda: tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf.
stop_gradient(
tf_util.max_pool2d(
net, [num_point, 1], padding='VALID', scope='maxpool') -
tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_2')))
elif pool == 'outer':
net = tf.cond(
is_outer, lambda: tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf.
stop_gradient(
tf_util.max_pool2d(
net, [num_point, 1], padding='VALID', scope='maxpool') -
tf_util.sum_pool2d(net, [num_point, 1],
padding='VALID',
scope='sumpool_2')), lambda: tf_util.
max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool'))
elif pool == 'both':
net = tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID',
scope='sumpool_1') + tf.stop_gradient(
tf_util.max_pool2d(
net, [num_point, 1], padding='VALID', scope='maxpool') -
tf_util.sum_pool2d(
net, [num_point, 1], padding='VALID', scope='sumpool_2'))
elif pool == 'mil':
net = tf_util.mil_pooling(net,
scope='milpool',
is_training=is_training)
elif pool == 'gate':
net = tf_util.gate_pooling(net,
scope='gatepool',
is_training=is_training)
elif pool == 'pma':
net = tf_util.pma_pooling(net,
scope='pmapool',
is_training=is_training)
elif pool == 'deepsym':
net = tf_util.deep_sym(net, scope='deepsym', is_training=is_training)
elif pool == 'fspool':
net = tf_util.fspool(net, scope='fspool')
elif pool == 'gate_v2':
net = tf_util.gate_pooling_v2(net,
scope='gatepool',
is_training=is_training)
# print(net.shape)
end_points['global'] = net
# print(net.shape)
net = tf.reshape(net, [batch_size, -1])
# net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
# scope='fc1', bn_decay=bn_decay)
# net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
# scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
activation_fn=tf.nn.relu,
scope='fc2',
bn_decay=bn_decay)
# net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
# scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def pointnet_sa_module_spec(xyz,
points,
npoint,
radius,
nsample,
mlp,
mlp2,
group_all,
is_training,
bn_decay,
scope,
bn=True,
pooling='max',
tnet_spec=None,
knn=False,
use_xyz=True,
spec_conv_type='mlp',
structure='spec',
useloc_covmat=True,
csize=None):
''' PointNet Set Abstraction (SA) Module
Input:
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor
npoint: int32 -- #points sampled in farthest point sampling
radius: float32 -- search radius in local region
nsample: int32 -- how many points in each local region
mlp: list of int32 -- output size for MLP on each point
mlp2: list of int32 -- output size for MLP on each region
group_all: bool -- group all points into one PC if set true, OVERRIDE
npoint, radius and nsample settings
use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features
# special to spectrual conv
useloc_covmat: use location xyz ONLY for local neighbourhood's covariance matrix computation; Set false to use all point features including xyz + network feats.
Return:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor
idx: (batch_size, npoint, nsample) int32 -- indices for local regions
'''
with tf.variable_scope(scope) as sc:
if group_all:
nsample = xyz.get_shape()[1].value
new_xyz, new_points, idx, grouped_xyz = sample_and_group_all(
xyz, points, use_xyz)
else:
new_xyz, new_points, idx, grouped_xyz = sample_and_group(
npoint, radius, nsample, xyz, points, tnet_spec, knn, use_xyz)
# grouped_xyz this is local xyz wrt to each center point
# dim B N K 3
# controls using what feature for covariance computation
if useloc_covmat:
local_cord = grouped_xyz
else:
local_cord = None
if structure == 'spec':
mlp_spec = mlp
new_points, UT = spec_conv2d(inputs=new_points,
num_output_channels=mlp_spec,
local_cord=local_cord,
bn=bn,
is_training=is_training,
scope='spec_conv%d' % (0),
bn_decay=bn_decay)
if structure == 'spec-modul':
mlp_spec = mlp
new_points, UT = spec_conv2d_modul(inputs=new_points,
num_output_channels=mlp_spec,
local_cord=local_cord,
bn=bn,
is_training=is_training,
scope='spec_conv%d' % (0),
bn_decay=bn_decay)
if pooling == 'avg':
new_points = tf_util.avg_pool2d(new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='avgpool1')
elif pooling == 'weighted_avg':
with tf.variable_scope('weighted_avg1'):
dists = tf.norm(grouped_xyz, axis=-1, ord=2, keep_dims=True)
exp_dists = tf.exp(-dists * 5)
weights = exp_dists / tf.reduce_sum(
exp_dists, axis=2,
keep_dims=True) # (batch_size, npoint, nsample, 1)
new_points *= weights # (batch_size, npoint, nsample, mlp[-1])
new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True)
elif pooling == 'max':
new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True)
elif pooling == 'min':
new_points = tf_util.max_pool2d(-1 * new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='minpool1')
elif pooling == 'max_and_avg':
avg_points = tf_util.max_pool2d(new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='maxpool1')
max_points = tf_util.avg_pool2d(new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='avgpool1')
new_points = tf.concat([avg_points, max_points], axis=-1)
elif pooling == 'hier_cluster_pool':
new_points = spec_hier_cluster_pool(new_points,
pool_method='max',
csize=csize)
if mlp2 is None: mlp2 = []
for i, num_out_channel in enumerate(mlp2):
new_points = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv_post_%d' % (i),
bn_decay=bn_decay)
new_points = tf.squeeze(new_points,
[2]) # (batch_size, npoints, mlp2[-1])
return new_xyz, new_points, idx
def pointnet_sa_module(xyz,
points,
xyz_feature,
npoint,
radius,
nsample,
mlp,
mlp2,
mlp3,
group_all,
is_training,
bn_decay,
scope,
bn=True,
pooling='max',
tnet_spec=None,
knn=False,
use_xyz=False,
end=False,
use_xyz_feature=False):
''' PointNet Set Abstraction (SA) Module
Input:
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor
npoint: int32 -- #points sampled in farthest point sampling
radius: float32 -- search radius in local region
nsample: int32 -- how many points in each local region
mlp: list of int32 -- output size for MLP on each point
mlp2: list of int32 -- output size for MLP on each region
group_all: bool -- group all points into one PC if set true, OVERRIDE
npoint, radius and nsample settings
use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features
Return:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor
idx: (batch_size, npoint, nsample) int32 -- indices for local regions
'''
with tf.variable_scope(scope) as sc:
batch_size = xyz.get_shape()[0].value
if group_all:
nsample = xyz.get_shape()[1].value
new_xyz, new_points, idx, grouped_xyz = sample_and_group_all(
xyz, points, use_xyz)
idx = tf.cast(idx, tf.int32)
########### additional xyz features
if use_xyz_feature:
xyz_feature = group_point(xyz_feature, idx)
e, grouped_xyz = PCA_decompose(grouped_xyz)
e = tf.tile(tf.expand_dims(e, 2), [1, 1, nsample, 1])
edge_feature = tf.concat(
[relative_pos_encoding(tf.abs(grouped_xyz)), e], axis=-1)
edge_feature = tf_util.conv2d(edge_feature,
mlp3[0], [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='xyz_feature_%d' % (0),
bn_decay=bn_decay)
edge_feature = tf_util.conv2d(edge_feature,
mlp3[1], [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='xyz_feature_%d' % (1),
bn_decay=bn_decay)
output_feature = tf.concat([xyz_feature, edge_feature], axis=-1)
if end == False:
xyz_feature = tf_util.conv2d(output_feature,
mlp3[-1], [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='xyz_feature2',
bn_decay=bn_decay)
# we can try sum and mean
xyz_feature = tf.reduce_max(xyz_feature,
axis=[2],
keep_dims=True,
name='maxpool')
xyz_feature = tf.squeeze(xyz_feature, [2])
new_points = tf.concat([new_points, output_feature], axis=-1)
for i, num_out_channel in enumerate(mlp):
new_points = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
padding='SAME',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv%d' % (i),
bn_decay=bn_decay)
if pooling == 'avg':
new_points = tf_util.avg_pool2d(new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='avgpool1')
elif pooling == 'weighted_avg':
with tf.variable_scope('weighted_avg1'):
dists = tf.norm(grouped_xyz, axis=-1, ord=2, keep_dims=True)
exp_dists = tf.exp(-dists * 5)
weights = exp_dists / tf.reduce_sum(
exp_dists, axis=2,
keep_dims=True) # (batch_size, npoint, nsample, 1)
new_points *= weights # (batch_size, npoint, nsample, mlp[-1])
new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True)
elif pooling == 'max':
new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True)
elif pooling == 'min':
new_points = tf_util.max_pool2d(-1 * new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='minpool1')
elif pooling == 'max_and_avg':
avg_points = tf_util.max_pool2d(new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='maxpool1')
max_points = tf_util.avg_pool2d(new_points, [1, nsample],
stride=[1, 1],
padding='VALID',
scope='avgpool1')
new_points = tf.concat([avg_points, max_points], axis=-1)
if mlp2 is None: mlp2 = []
for i, num_out_channel in enumerate(mlp2):
new_points = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv_post_%d' % (i),
bn_decay=bn_decay)
new_points = tf.squeeze(new_points,
[2]) # (batch_size, npoints, mlp2[-1])
return new_xyz, new_points, idx, xyz_feature
