def input_transform_net(points, is_training, name, dim=3, reused=None):
""" Input (XYZ) Transform Net, input is BxNx3 points data
Return:
Transformation matrix of size 3xK """
pts_num = points.get_shape()[1].value
batch_size = points.get_shape()[0].value
net = pf.dense(points, 64, name + 'mlp1', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = pf.dense(net, 128, name + 'mlp2', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = pf.dense(net, 1024, name + 'mlp3', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = tf.expand_dims(net, axis=-1, name=name+'expand_dim')
net = pf.max_pool2d(net, [pts_num, 1], stride=[1, 1], name=name + 'pooling')
net = tf.reshape(net, [batch_size, 1024])
net = pf.dense(net, 128, name + 'mlp4', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = pf.dense(net, 64, name + 'mlp5', is_training, activation_fn=tf.nn.relu, reuse=reused)
with tf.variable_scope(name + 'xyz_transform', reuse=reused):
assert(dim == 3)
weights = tf.get_variable('weights', [64, 3*dim],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases = tf.get_variable('biases', [3*dim],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases += tf.constant([1, 0, 0, 0, 1, 0, 0, 0, 1], dtype=tf.float32)
transform = tf.matmul(net, weights)
transform = tf.nn.bias_add(transform, biases)
transform = tf.reshape(transform, [batch_size, 3, dim])
return transform
def __call__(self, point_cloud):
'''
input:
point_cloud: BxNx3
output:
latent_code: Bxlatent_code_dim
'''
with tf.variable_scope(self.name):
#batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
#sort_mtd = 'cxyz'
sort_mtd= None
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = point_conv_module(l0_xyz, l0_points, npoint=512, c_fts_out=128, k_neighbors=64, d_rate=1, is_training=is_training, scope='conv_layer_1', sorting_method=sort_mtd)
l2_xyz, l2_points, l2_indices = point_conv_module(l1_xyz, l1_points, npoint=128, c_fts_out=256, k_neighbors=32, d_rate=1, is_training=is_training, scope='conv_layer_2', sorting_method=sort_mtd)
l3_xyz, l3_points, l3_indices = point_conv_module(l2_xyz, l2_points, npoint=24, c_fts_out=512, k_neighbors=8, d_rate=1, is_training=is_training, scope='conv_layer_3', sorting_method=sort_mtd)
# fc
fc = pf.dense(l5_points, 1024, 'fc1', self.is_training)
fc = pf.dense(fc, 512, 'fc2', self.is_training)
latent_code = pf.dense(fc, self.latent_code_dim, 'fc3', self.is_training)
# B x latent_code_dim
return latent_code
def input_qtransform_net(points, is_training, name, reused=None):
batch_size = points.get_shape()[0].value
point_num = points.get_shape()[1].value
net = pf.dense(points, 64, name + 'mlp1', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = pf.dense(net, 128, name + 'mlp2', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = pf.dense(net, 1024, name + 'mlp3', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = tf.expand_dims(net, axis=-1, name=name + 'expand_dim')
net = pf.max_pool2d(net, [point_num, 1], stride=[1, 1], name=name+'pooling')
net = tf.reshape(net, [batch_size, 1, 1024])
net = pf.dense(net, 128, name + 'mlp4', is_training, activation_fn=tf.nn.relu, reuse=reused)
net = pf.dense(net, 64, name + 'mlp5', is_training, activation_fn=tf.nn.relu, reuse=reused)
with tf.variable_scope(name+'q_transform', reuse=reused):
weights = tf.get_variable('weights', [64, 4], initializer=tf.constant_initializer(0.0), dtype=tf.float32)
biases = tf.get_variable('biases', [4], initializer=tf.constant_initializer(0.0), dtype=tf.float32)
biases += tf.constant([1, 0, 0, 0], dtype=tf.float32)
weights = tf.expand_dims(weights, 0)
weights = tf.tile(weights, [batch_size, 1, 1])
transform = tf.matmul(net, weights)
# [batch_size, 1, 4] each shape has a trans matrix
transform = tf.nn.bias_add(transform, biases)
qtransform = batch_quat_to_rotmat(transform)
return qtransform
def seg_shape(self, feats, parts_num, name, is_training, is_resued=None):
net = pf.dense(feats,
1024,
name + 'mlp1',
is_training,
reuse=is_resued,
activation_fn=tf.nn.relu)
net = pf.dense(net,
512,
name + 'mlp2',
is_training,
reuse=is_resued,
activation_fn=tf.nn.relu)
net = pf.dense(net,
256,
name + 'mlp3',
is_training,
reuse=is_resued,
activation_fn=tf.nn.relu)
net = pf.dense(net,
128,
name + 'mlp4',
is_training,
reuse=is_resued,
activation_fn=tf.nn.relu)
net = pf.dense(net,
parts_num,
name + 'mlp5',
is_training,
reuse=is_resued,
activation_fn=None)
return net
def points_category(self,
shape_fts,
category_num,
is_training,
name,
is_resused=None):
net = pf.dense(shape_fts,
512,
name + 'mlp1',
is_training,
reuse=is_resused,
activation_fn=tf.nn.relu)
net = tf.layers.dropout(net,
0.3,
training=is_training,
name=name + 'dp1')
net = pf.dense(net,
256,
name + 'mlp2',
is_training,
reuse=is_resused,
activation_fn=tf.nn.relu)
net = tf.layers.dropout(net,
0.3,
training=is_training,
name=name + 'dp2')
net = pf.dense(net,
category_num,
name + 'mlp3',
is_training,
reuse=is_resused,
activation_fn=None)
return net
def points_deform(self, feats, name, is_training, is_reused=None):
bottleneck_size = feats.get_shape()[-1].value
feats = pf.dense(feats,
bottleneck_size,
name + 'mlp1',
is_training,
reuse=is_reused,
activation_fn=tf.nn.relu)
feats = pf.dense(feats,
bottleneck_size // 2,
name + 'mlp2',
is_training,
reuse=is_reused,
activation_fn=tf.nn.relu)
feats = pf.dense(feats,
bottleneck_size // 4,
name + 'mlp3',
is_training,
reuse=is_reused,
activation_fn=tf.nn.relu)
feats = pf.dense(feats,
3,
name + 'mlp4',
is_training,
reuse=is_reused,
activation_fn=tf.nn.tanh)
points_deform = 2 * feats # why ？？？
return points_deform
def input_transform_net(point_cloud, is_training,N, PP,KK=3):
""" Input (XYZ) Transform Net, input is BxNx3 gray image
Return:
Transformation matrix of size 3xK """
input_image = tf.expand_dims(point_cloud, -1)
print(input_image)
net = pf.conv2d(input_image, 64, 'tconv1', is_training,(1,3))
net = pf.dense(net, 128, 'tconv2', is_training)
net = pf.dense(net, 1024, 'tconv3', is_training)
print(net)
net = pf.max_pool2d(net,[1024,1],'tmaxpool')
net = pf.dense(net, 512, 'tfc1',is_training)
net = pf.dense(net, 256, 'tfc2',is_training)
net = tf.reshape(net, (N, -1))
with tf.variable_scope('transform_XYZ') as sc:
assert(KK==3)
weights = tf.get_variable('weights', (256, 3*KK),
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases = tf.get_variable('biases', (3*KK),
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
biases += tf.constant([1,0,0,0,1,0,0,0,1], dtype=tf.float32)
transform = tf.matmul(net, weights)
transform = tf.nn.bias_add(transform, biases)
transform = tf.reshape(transform, (N, 3, KK))
return transform
def point_feat_extract(self, points, name, is_training, is_trans, is_reused=None):
pts_num = points.get_shape()[1].value
if is_trans == 'q_trans':
transform_matrix = input_qtransform_net(points, is_training, name+'rotation', reused = is_reused)
points_t = tf.matmul(points, transform_matrix)
else:
if is_trans == 'trans':
transform_matrix = input_transform_net(points, is_training, name+'rotation', reused = is_reused)
points_t = tf.matmul(points, transform_matrix)
else:
points_t = points
net = pf.dense(points_t, 64, name+'mlp1', is_training, reuse=is_reused, activation_fn=tf.nn.relu)
net = pf.dense(net, 128, name+'mlp2', is_training, reuse=is_reused, activation_fn=tf.nn.relu)
net = pf.dense(net, 256, name+'mlp3', is_training, reuse=is_reused, activation_fn=tf.nn.relu)
net = pf.dense(net, 1024, name+'mlp4', is_training, reuse=is_reused, activation_fn=tf.nn.relu)
pfts = net
net = tf.expand_dims(net, axis=-1, name=name+'extend_input')
net = pf.max_pool2d(net, [pts_num, 1], stride=[1, 1], name=name+'pooling')
net_tile = tf.tile(tf.squeeze(net, axis=-1), [1, pts_num, 1])
concat_pfts = tf.concat([pfts, net_tile], axis=-1)
if is_trans is not 'no_trans':
return concat_pfts, tf.squeeze(net, axis=[1, 3]), transform_matrix, points_t
else:
return concat_pfts, tf.squeeze(net, axis=[1, 3]), points_t
def pointnet1(point_cloud, is_training, N, PP):
transform = input_transform_net(point_cloud, is_training, N, PP, KK=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1) # (N,PP,3,1)
net = pf.conv2d(input_image, 64, 'p1_conv1', is_training,(1,3)) # (N,PP,1,64)
net = pf.dense(net, 64, 'p1_mlp1', is_training) # (N,PP,1,64)
net = pf.dense(net, 64, 'p1_mlp2', is_training) # (N,PP,1,64)
net = pf.dense(net, 128, 'p1_mlp3', is_training) # (N,PP,1,128)
net = pf.dense(net, 1024, 'p1_mlp4', is_training) # (N,PP,1,1024)
net1 = net
net = pf.max_pool2d(net,[PP,1],'p1_maxpool') # (N,1,1,1024)
net = tf.squeeze(net, axis=[2])
net1 = tf.squeeze(net1, axis=[2])
return net,net1
def __init__(self, points, features, is_training, setting):
PointCNN.__init__(self, points, features, is_training, setting)
self.point_features = self.layer_fts[1]
fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keepdims=True, name='fc_mean')
self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean)
self.logits = pf.dense(self.fc_layers[-1], setting.num_class, 'logits',
is_training, with_bn=False, activation=None)
def __init__(self, points, features, is_training, setting):
PointCNN.__init__(self, points, features, is_training, setting)
self.logits = pf.dense(self.fc_layers[-1],
setting.num_class,
'logits',
is_training,
with_bn=False,
activation=None)
def __init__(self, points, features, is_training, setting):
PointCNN.__init__(self, points, features, is_training, setting)
with tf.variable_scope(setting.network_name):
batch_size = points.get_shape()[0].value
fc_flatten = tf.reshape(self.fc_layers[-1], [batch_size, -1])
fc_flatten = tf.concat([fc_flatten, features[:, 0, :]], axis=1)
fc1 = pf.dense(fc_flatten, 512, 'extra_fc_1', is_training)
# fc1_drop = tf.layers.dropout(fc1, 0.0, training=is_training, name='extra_fc_1_drop')
# self.fc_layers.append(fc1_drop)
fc2 = pf.dense(fc1, 256, 'extra_fc_2', is_training)
self.output = pf.dense(fc2,
3 + NUM_HEADING_BIN * 2 +
NUM_SIZE_CLUSTER * 4,
'output',
is_training,
with_bn=False,
activation=None)
def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier,
sorting_method=None, with_global=False):
if D == 1:
_, indices = pf.knn_indices_general(qrs, pts, K, True)
else:
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input')
if with_X_transformation:
######################## X-transformation #########################
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None)
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training)
return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def point_conv_transpose_fc_no_skip(pts_in,
fts_in,
indices,
np_out,
K,
C,
is_training,
activation=tf.nn.relu,
bn=True,
tag='point_deconv'):
'''
NOTE: no skipped link of output points and features
pts_in: xyz points, Tensor, (batch_size, npoint, 3)
fts_in: features, Tensor, (batch_size, npoint, channel_fts)
np_out: number of points that is expected
K: neighbor size
C: output feature channel number
'''
with tf.variable_scope(tag):
batch_size = fts_in.shape[0]
n_pts_in = fts_in.shape[1]
if pts_in is None:
nn_fts_input = fts_in # (N, P, C_fts_in)
#print('Error: fts_in is None.')
else:
nn_fts_input = tf.concat(
[pts_in, fts_in], axis=-1,
name='_nn_fts_input') # (N, P, C_fts_in+3)
nn_fts_input = tf.expand_dims(nn_fts_input,
axis=2) # (N, P, 1, C_fts_in)
# using fc
fts_fc = pf.dense(nn_fts_input, K * C, 'fc_fts_prop',
is_training) # (N, P, 1, K*C)
fts_fc = tf.reshape(fts_fc,
(batch_size, n_pts_in, K, C)) # (N, P, K, C)
fts_pts_conv = pf.conv2d(
fts_all_concat,
C,
tag + '_fts_pts_conv1D',
is_training,
kernel_size=(1, 1),
strides=(1, 1),
with_bn=bn,
activation=activation) # (N, P, K, C+3_ch_skip?) -> (N, P, K, C)
# summing up feature for each point
fts_dense = tf.scatter_nd(
tf.reshape(indices, (-1, 2)), tf.reshape(fts_pts_conv, (-1, C)),
(batch_size, n_pts_out, C)) # (N, N_PTS_OUT, c)
return fts_dense
def __init__(self, points, features, is_training, setting):
PointCNN_SEG.__init__(self, points, features, is_training, setting)
###Classification
self.classification_logits = pf.dense(
self.fc_layers_classification[-1],
setting.num_class,
'logits_classification',
is_training,
with_bn=False,
activation=None)
###Segmentation Mask
self.segmentation_logits = pf.dense(self.fc_layers_segmentation[-1],
2,
'logits_segmentation',
is_training,
with_bn=False,
activation=None)
def pointcnn_xupconv_module(xyz, points, dense_xyz, dense_points_skip,
c_fts_out, c_x, k_neighbors, d_rate, is_training,
depth_multiplier, with_global, scope):
'''
input:
xyz: TF tensor, input point clouds coords, B x N x 3
points: TF tensor, input point clouds features, B x N x fts_channel
dense_xyz: TF tensor, representative points previously sampled, B x npoint x 3
dense_points_skip: skip link from previously generated features
c_fts_out: int32, output channels number
c_x: int32, channels number of lifted features for x-transformation matrix
k_neighbors: int32, neighbor size
d_rate: int32, dilation rate
is_training: TF tensor, flag indicating training status
output:
new_xyz: TF tensor, output point clouds coords, B x npoint x 3
new_points: TF tensor, output point clouds features, B x npoint x c_fts_out
'''
with tf.variable_scope(scope):
new_xyz = dense_xyz
npoint = new_xyz.shape[1]
new_points = xconv(xyz,
points,
new_xyz,
scope,
xyz.shape[0],
k_neighbors,
d_rate,
npoint,
c_fts_out,
c_x,
is_training,
True,
depth_multiplier,
sorting_method=None,
with_global=with_global)
if dense_points_skip is not None:
points_concat = tf.concat([new_points, dense_points_skip],
axis=-1,
name=scope + '_skip_concat')
new_points = pf.dense(points_concat, c_fts_out, scope + '_fuse',
is_training)
return new_xyz, new_points
def __init__(self, points, features, is_training, setting):
PointCNN.__init__(self, points, features, is_training, setting)
fc_mean = tf.reduce_mean(self.fc_layers[-1],
axis=1,
keep_dims=True,
name='fc_mean')
self.feature_list = tf.reshape(self.feature_list, [128, 61440])
self.feature_list_A = self.feature_list[0:64]
self.feature_list_B = self.feature_list[64:128]
self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1],
lambda: fc_mean) #最后一层连接
self.logits = pf.dense(self.fc_layers[-1],
setting.num_class,
'logits',
is_training,
with_bn=False,
activation=None)
def get_model(layer_pts, is_training, RIconv_params, RIdconv_params, fc_params, sampling='fps', weight_decay=0.0, bn_decay=None, part_num=50):
if sampling == 'fps':
sys.path.append(os.path.join(BASE_DIR, 'tf_ops/sampling'))
from tf_sampling import farthest_point_sample, gather_point
layer_fts_list = [None]
layer_pts_list = [layer_pts]
for layer_idx, layer_param in enumerate(RIconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
P = layer_param['P']
C = layer_param['C']
# qrs = layer_pts if P == -1 else layer_pts[:,:P,:] # (N, P, 3)
if P == -1:
qrs = layer_pts
else:
if sampling == 'fps':
qrs = gather_point(layer_pts, farthest_point_sample(P, layer_pts))
elif sampling == 'random':
qrs = tf.slice(layer_pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3)
else:
print('Unknown sampling method!')
exit()
layer_fts= RIConv(layer_pts_list[-1], layer_fts_list[-1], qrs, is_training, tag, K, D, P, C, True, bn_decay)
layer_pts = qrs
layer_pts_list.append(qrs)
layer_fts_list.append(layer_fts)
if RIdconv_params is not None:
fts = layer_fts_list[-1]
for layer_idx, layer_param in enumerate(RIdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
pts_layer_idx = layer_param['pts_layer_idx'] # 2 1 0
qrs_layer_idx = layer_param['qrs_layer_idx'] # 1 0 -1
pts = layer_pts_list[pts_layer_idx + 1]
qrs = layer_pts_list[qrs_layer_idx + 1]
fts_qrs = layer_fts_list[qrs_layer_idx + 1]
C = fts_qrs.get_shape()[-1].value if fts_qrs is not None else C//2
P = qrs.get_shape()[1].value
layer_fts= RIConv(pts, fts, qrs, is_training, tag, K, D, P, C, True, bn_decay)
if fts_qrs is not None: # this is for last layer
fts_concat = tf.concat([layer_fts, fts_qrs], axis=-1, name=tag + 'fts_concat')
fts = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training)
else:
fts = layer_fts
for layer_idx, layer_param in enumerate(fc_params):
C = layer_param['C']
dropout_rate = layer_param['dropout_rate']
layer_fts = pf.dense(layer_fts, C, 'fc{:d}'.format(layer_idx), is_training)
layer_fts = tf.layers.dropout(layer_fts, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx))
logits_seg = pf.dense(layer_fts, part_num, 'logits', is_training, with_bn=False, activation=None)
return logits_seg
def __init__(self, points, features, num_class, is_training, setting,
task):
xconv_params = setting.xconv_params
fc_params = setting.fc_params
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
if setting.with_fps:
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
C_fts = xconv_params[0][-1] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K, D, P, C = layer_param
# get k-nearest points
pts = self.layer_pts[-1]
fts = self.layer_fts[-1]
if P == -1:
qrs = points
else:
if setting.with_fps:
qrs = tf_sampling.gather_point(
pts, tf_sampling.farthest_point_sample(P,
pts)) # (N,P,3)
else:
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1][-1]
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method)
self.layer_fts.append(fts_xconv)
if task == 'segmentation':
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K, D, pts_layer_idx, qrs_layer_idx = layer_param
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[
pts_layer_idx +
1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
_, _, P, C = xconv_params[qrs_layer_idx]
_, _, _, C_prev = xconv_params[pts_layer_idx]
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C,
C_pts_fts, is_training,
with_X_transformation, depth_multiplier,
sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs],
axis=-1,
name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse',
is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.fc_layers[-1], channel_num,
'fc{:d}'.format(layer_idx), is_training)
fc_drop = tf.layers.dropout(fc,
drop_rate,
training=is_training,
name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
logits = pf.dense(self.fc_layers[-1],
num_class,
'logits',
is_training,
with_bn=False,
activation=None)
if task == 'classification':
logits_mean = tf.reduce_mean(logits,
axis=1,
keep_dims=True,
name='logits_mean')
self.logits = tf.cond(is_training, lambda: logits,
lambda: logits_mean)
elif task == 'segmentation':
self.logits = logits
else:
print('Unknown task!')
exit()
self.probs = tf.nn.softmax(self.logits, name='probs')
def __init__(self, points, features, is_training, setting):
xconv_params = setting.xconv_params
fc_params = setting.fc_params
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
if setting.sampling == 'fps':
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
# expand feature dimentions
features = tf.reshape(features, (N, -1, setting.data_dim - 3),
name='features_reshape')
C_fts = xconv_params[0]['C'] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K = layer_param['K'] # neighborhood size
D = layer_param['D'] # dilate rate
P = layer_param['P'] # representative point number
C = layer_param['C'] # output channel number
links = layer_param[
'links'] # e.g. [-1,-2] will tell the current layer to receive inputs from the previous two layers
if setting.sampling != 'random' and links:
print(
'Error: flexible links are supported only when random sampling is used!'
)
exit()
# get k-nearest points
pts = self.layer_pts[-1] # get the end one in list (last layer)
fts = self.layer_fts[-1]
# if P == -1 or p equals to last layer's p , than no sampling
if P == -1 or (layer_idx > 0
and P == xconv_params[layer_idx - 1]['P']):
qrs = self.layer_pts[-1]
else:
# for segmentation task
if setting.sampling == 'fps':
# fps_indices: [[0_0, 0_1, ...., 0_P-1], ...., [N-1_0, N-1_1, ...., N-1_P-1]] (NxP)
fps_indices = tf_sampling.farthest_point_sample(P, pts)
# replicating a index(0...N) for each output poit(P) -> (N, P, 1)
# batch_indices: [[[0],[0],...,[0]], ..., [[N-1],[N-1],...,[N-1]]]
batch_indices = tf.tile(
tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1))
# tf.expand_dims(fps_indices, -1): [[[0_0], [0_1], ..., [0_P-1]], ... , [[N-1_0], [N-1_1], ... , [N-1_P-1]]]
# indices: [[[0, 0_0], [0, 0_1], ..., [0, 0_P-1]], ..., [[N-1, N-1_0], [N-1, N-1_1], ... , [N-1, N-1_P-1]]]
indices = tf.concat(
[batch_indices,
tf.expand_dims(fps_indices, -1)],
axis=-1)
qrs = tf.gather_nd(pts, indices,
name=tag + 'qrs') # (N, P, 3)
elif setting.sampling == 'ids':
indices = pf.inverse_density_sampling(pts, K, P)
qrs = tf.gather_nd(pts, indices)
# for classification tasks
elif setting.sampling == 'random':
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
else:
print('Unknown sampling method!')
exit()
self.layer_pts.append(qrs)
if layer_idx == 0:
# lift each point coordinates into C_pts_fts dimensional space
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1]['C']
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
# with global feature at last layer
with_global = (setting.with_global
and layer_idx == len(xconv_params) - 1)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method, with_global)
fts_list = []
# receive inputs from previous layers
for link in links:
fts_from_link = self.layer_fts[link]
if fts_from_link is not None:
fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1),
name=tag + 'fts_slice_' + str(-link))
fts_list.append(fts_slice)
if fts_list:
fts_list.append(fts_xconv)
self.layer_fts.append(
tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat'))
else:
self.layer_fts.append(fts_xconv)
if hasattr(setting, 'xdconv_params'):
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
pts_layer_idx = layer_param['pts_layer_idx']
qrs_layer_idx = layer_param['qrs_layer_idx']
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[
pts_layer_idx +
1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
P = xconv_params[qrs_layer_idx]['P']
C = xconv_params[qrs_layer_idx]['C']
C_prev = xconv_params[pts_layer_idx]['C']
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C,
C_pts_fts, is_training,
with_X_transformation, depth_multiplier,
sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs],
axis=-1,
name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse',
is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
C = layer_param['C']
dropout_rate = layer_param['dropout_rate']
fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx),
is_training)
fc_drop = tf.layers.dropout(fc,
dropout_rate,
training=is_training,
name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
def ficonv(pts,
fts,
qrs,
tag,
N,
K1,
mm,
sigma,
scale,
K,
D,
P,
C,
C_pts_fts,
kernel_num,
is_training,
with_kernel_registering,
with_kernel_shape_comparison,
with_point_transformation,
with_feature_transformation,
with_learning_feature_transformation,
kenel_initialization_method,
depth_multiplier,
sorting_method=None,
with_global=False):
Dis, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + 'nn_pts_local') # (N, P, K, 3)
if with_point_transformation or with_feature_transformation:
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None)
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
if with_point_transformation:
if with_learning_feature_transformation:
nn_pts_local = tf.matmul(X_2_KK, nn_pts_local)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
else:
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
nn_pts_local = tf.matmul(X_2_KK, nn_pts_local)
else:
if with_learning_feature_transformation:
nn_pts_local_ = tf.matmul(X_2_KK,
nn_pts_local,
name=tag + 'nn_pts_local_')
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local_, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
else:
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
P1 = tf.shape(nn_pts_local)[1]
dim1 = 3
if with_kernel_registering:
######################## preparing #########################
if with_feature_transformation:
nn_fts_input = tf.matmul(X_2_KK, nn_fts_input)
r_data = tf.reduce_sum(nn_pts_local * nn_pts_local,
axis=3,
keep_dims=True,
name=tag + 'kernel_pow')
######################## kernel-registering #########################
shape_id = 0
if kenel_initialization_method == 'random':
kernel_shape = tf.Variable(tf.random_uniform([K1, dim1],
minval=-0.5,
maxval=0.5,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
else:
kernel_shape = tf.Variable(tf.random_normal([K1, dim1],
mean=0.0,
stddev=1.0,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
kernel_shape_dis = tf.sqrt(tf.reduce_sum(kernel_shape * kernel_shape,
axis=1),
name=tag + 'kernel_shape_dis' +
str(shape_id))
kernel_shape_normal = scale * tf.div(
kernel_shape,
tf.reduce_max(kernel_shape_dis),
name=tag + 'kernel_shape_normal' + str(shape_id))
r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal,
axis=1,
keep_dims=True,
name=tag + 'kernel_pow' + str(shape_id))
reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1],
name=tag + 'reshape_kernel' + str(shape_id))
m = tf.reshape(tf.matmul(reshape_data,
tf.transpose(kernel_shape_normal)),
[N, P1, K, K1],
name=tag + 'mm' + str(shape_id))
dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel),
perm=[0, 1, 3, 2],
name=tag + 'dis_matrix' + str(shape_id))
coef_matrix = tf.exp(tf.div(-dis_matrix, sigma),
name=tag + 'coef_matrix' + str(shape_id))
#coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id))
if with_kernel_shape_comparison:
coef_global = tf.reduce_sum(
coef_matrix, axis=[2, 3], keep_dims=True) / K
coef_normal = coef_global * tf.div(
coef_matrix,
tf.reduce_sum(coef_matrix, axis=3, keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
else:
coef_normal = tf.div(coef_matrix,
tf.reduce_sum(coef_matrix,
axis=3,
keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
fts_X = tf.matmul(coef_normal,
nn_fts_input,
name=tag + 'fts_X' + str(shape_id))
###################################################################
fts_conv = pf.separable_conv2d(fts_X,
math.ceil(mm * C / kernel_num),
tag + 'fts_conv' + str(shape_id),
is_training, (1, K1),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv,
axis=2,
name=tag + 'fts_conv_3d' + str(shape_id))
for shape_id in range(kernel_num - 1):
shape_id = shape_id + 1
if kenel_initialization_method == 'random':
kernel_shape = tf.Variable(tf.random_uniform([K1, dim1],
minval=-0.5,
maxval=0.5,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
else:
kernel_shape = tf.Variable(tf.random_normal([K1, dim1],
mean=0.0,
stddev=1.0,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
kernel_shape_dis = tf.sqrt(
tf.reduce_sum(kernel_shape * kernel_shape, axis=1),
name=tag + 'kernel_shape_dis' + str(shape_id))
kernel_shape_normal = scale * tf.div(
kernel_shape,
tf.reduce_max(kernel_shape_dis),
name=tag + 'kernel_shape_normal' + str(shape_id))
r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal,
axis=1,
keep_dims=True,
name=tag + 'kernel_pow' + str(shape_id))
reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1],
name=tag + 'reshape_kernel' +
str(shape_id))
m = tf.reshape(tf.matmul(reshape_data,
tf.transpose(kernel_shape_normal)),
[N, P1, K, K1],
name=tag + 'mm' + str(shape_id))
dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel),
perm=[0, 1, 3, 2],
name=tag + 'dis_matrix' + str(shape_id))
coef_matrix = tf.exp(tf.div(-dis_matrix, sigma),
name=tag + 'coef_matrix' + str(shape_id))
#coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id))
if with_kernel_shape_comparison:
coef_global = tf.reduce_sum(
coef_matrix, axis=[2, 3], keep_dims=True) / K
coef_normal = coef_global * tf.div(
coef_matrix,
tf.reduce_sum(coef_matrix, axis=3, keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
else:
coef_normal = tf.div(coef_matrix,
tf.reduce_sum(coef_matrix,
axis=3,
keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
fts_X = tf.matmul(coef_normal,
nn_fts_input,
name=tag + 'fts_X' + str(shape_id))
###################################################################
fts_conv = pf.separable_conv2d(fts_X,
math.ceil(mm * C / kernel_num),
tag + 'fts_conv' + str(shape_id),
is_training, (1, K1),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.concat(
[fts_conv_3d, tf.squeeze(fts_conv, axis=2)],
axis=-1,
name=tag + 'fts_conv_3d' + str(shape_id))
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global',
is_training)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def __init__(self, points, features, is_training, setting):
xconv_params = setting.xconv_params
fc_params = setting.fc_params
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
if setting.sampling == 'fps':
import sys
#sys.path.append("/home/hasan/data/paper2/PointCNN/")
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
features = tf.reshape(features, (N, -1, setting.data_dim - 3),
name='features_reshape')
C_fts = xconv_params[0]['C'] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
P = layer_param['P']
C = layer_param['C']
links = layer_param['links']
if setting.sampling != 'random' and links:
print(
'Error: flexible links are supported only when random sampling is used!'
)
exit()
# get k-nearest points
pts = self.layer_pts[-1]
fts = self.layer_fts[-1]
if P == -1 or (layer_idx > 0
and P == xconv_params[layer_idx - 1]['P']):
qrs = self.layer_pts[-1]
else:
if setting.sampling == 'fps':
fps_indices = tf_sampling.farthest_point_sample(P, pts)
batch_indices = tf.tile(
tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1))
indices = tf.concat(
[batch_indices,
tf.expand_dims(fps_indices, -1)],
axis=-1)
qrs = tf.gather_nd(pts, indices,
name=tag + 'qrs') # (N, P, 3)
elif setting.sampling == 'ids':
indices = pf.inverse_density_sampling(pts, K, P)
qrs = tf.gather_nd(pts, indices)
elif setting.sampling == 'random':
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
else:
print('Unknown sampling method!')
exit()
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1]['C']
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
with_global = (setting.with_global
and layer_idx == len(xconv_params) - 1)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method, with_global)
fts_list = []
for link in links:
fts_from_link = self.layer_fts[link]
if fts_from_link is not None:
fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1),
name=tag + 'fts_slice_' + str(-link))
fts_list.append(fts_slice)
if fts_list:
fts_list.append(fts_xconv)
self.layer_fts.append(
tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat'))
else:
self.layer_fts.append(fts_xconv)
if hasattr(setting, 'xdconv_params'):
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
pts_layer_idx = layer_param['pts_layer_idx']
qrs_layer_idx = layer_param['qrs_layer_idx']
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[
pts_layer_idx +
1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
P = xconv_params[qrs_layer_idx]['P']
C = xconv_params[qrs_layer_idx]['C']
C_prev = xconv_params[pts_layer_idx]['C']
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C,
C_pts_fts, is_training,
with_X_transformation, depth_multiplier,
sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs],
axis=-1,
name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse',
is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
C = layer_param['C']
dropout_rate = layer_param['dropout_rate']
fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx),
is_training)
fc_drop = tf.layers.dropout(fc,
dropout_rate,
training=is_training,
name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
def xconv(pts,
fts,
qrs,
tag,
N,
K,
D,
P,
C,
depth_multiplier,
is_training,
C_pts_fts,
with_x_transformation=True,
sorting_method=None,
with_global=False):
###xconv(pts,fts,qrs,tag,N,K,D,P,C,C_pts_fts,is_training,with_X_transformation,depth_multipiler,sorting_method,with_global)
# @params:
# N: Number of output points
# K: Number of nearest neighbot
# D: dilation rate
# C: Number of output channels
# P:the representative point number in the output, -1 means all input points are output representative points
# x_transformation: replace max_pooling in PointNet
# sorting_method:
# with_global
# pts:Input point cloud
# qrs:queries
# fts:features
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') #(N,P,K,3)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') #(N,P,1,3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + 'nn_pts_local') #(N,P,K,3)
#Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
#X_transformation
if with_x_transformation:
############################X_transformation#######################################################
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None)
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
#####################################################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global',
is_training)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def Invariance_Transformation_Net(point_cloud, features, is_training,
invarians_trans_param):
xconv_params = invarians_trans_param.xconv_params
fc_params = invarians_trans_param.fc_params
with_X_transformation = invarians_trans_param.with_X_transformation
sorting_method = invarians_trans_param.sorting_method
#N=point_cloud.get_shape()[1].value
N = point_cloud.get_shape()[0].value
if invarians_trans_param.sampling == 'fps':
import tf_sampling
layer_pts = [point_cloud]
if features is None:
layer_fts = [features]
else:
features = tf.reshape(features,
(N, -1, invarians_trans_param.data_dim - 3),
name='features_reshape')
C_fts = xconv_params[0]['C'] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(
layer_idx +
1) + '_' #####xconv_1_ #####xconv_2_ #####xconv_3_ #####xconv_4_
K = layer_param['K']
D = layer_param['D']
P = layer_param['P']
C = layer_param['C']
links = layer_param['links'] ## type(layer_param) is dict
#get k-nearest points
pts = layer_pts[-1]
fts = layer_fts[-1]
if P == -1 or (layer_idx > 0
and P == xconv_params[layer_idx - 1]['P']):
qrs = layer_pts[-1]
else:
if invarians_trans_param.sampling == 'fps':
fps_indices = tf_sampling.farthest_point_sample(P, pts)
batch_indices = tf.tile(tf.reshape(tf.range(N), (-1, 1, 1)),
(1, P, 1))
indices = tf.concat(
[batch_indices,
tf.expand_dims(fps_indices, -1)], axis=-1)
qrs = tf.gather_nd(pts, indices, name=tag + 'qrs') ### (N,P,3)
print(tf.shape(qrs))
elif invarians_trans_param.sampling == 'ids':
indices = pf.inverse_density_sampling(pts, K, P)
qrs = tf.gather_nd(pts, indices)
elif invarians_trans_param == 'random':
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') ### (N,P,3)
else:
print('unknown sampling method')
exit()
layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multipiler = 4
else:
C_prev = xconv_params[layer_idx - 1]['C']
C_pts_fts = C_prev // 4
depth_multipiler = math.ceil(C / C_prev)
with_global = (invarians_trans_param.with_global
and layer_idx == len(xconv_params) - 1)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation, depth_multipiler,
sorting_method, with_global)
fts_list = []
for link in links:
fts_from_link = layer_fts[link]
if fts_from_link is not None:
fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1),
name=tag + 'fts_slice' + str(-link))
fts_list.append(fts_slice)
if fts_list:
fts_list.append(fts_xconv)
layer_fts.append(
tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat'))
else:
layer_fts.append(fts_xconv)
if hasattr(invarians_trans_param, 'xdconv_params'):
for layer_idx, layer_param in enumerate(
invarians_trans_param.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
pts_layer_idx = layer_param['pts_layer_idx']
qrs_layer_idx = layer_param['qrs_layer_idx']
pts = layer_pts[pts_layer_idx + 1]
fts = layer_fts[pts_layer_idx +
1] if layer_idx == 0 else layer_fts[-1]
qrs = layer_pts[qrs_layer_idx + 1]
fts_qrs = layer_fts[qrs_layer_idx + 1]
P = xconv_params[qrs_layer_idx]['P']
C = xconv_params[qrs_layer_idx]['C']
C_prev = xconv_params[pts_layer_idx]['C']
C_pts_fts = C_prev // 4
depth_multipiler = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multipiler, sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs],
axis=-1,
name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training)
layer_pts.append(qrs)
layer_fts.append(fts_fuse)
fc_layers = layer_fts[-1]
for layer_idx, layer_param in enumerate(fc_params):
C = layer_param['C']
dropout_rate = layer_param['dropout_rate']
fc = pf.dense(fc_layers[-1], C, 'fc{:d}'.format(layer_idx),
is_training)
fc_drop = tf.layers.dropout(fc,
dropout_rate,
training=is_training,
name='fc{:d}_drop'.format(layer_idx))
fc_layers.append(fc_drop)
return fc_layers
def main():
num_class = 6
pts_fts = tf.placeholder(tf.float32, [2,1,7])
labels = tf.placeholder(tf.int32, [2,])
global_step = tf.Variable(0, trainable=False, name='global_step')
is_training = tf.placeholder(tf.bool, name='is_training')
features_hd = pts_fts#pf.dense(pts_fts, 16, 'features_hd', is_training)
#fc = pf.dense(features_hd, 128, 'fc', is_training, with_bn=True)
dense = tf.layers.dense(features_hd, units=16, activation=tf.nn.elu,
kernel_initializer=tf.glorot_uniform_initializer(),
kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=1.0))
fc = tf.layers.batch_normalization(dense, momentum=0.9, training=is_training,
beta_regularizer=tf.contrib.layers.l2_regularizer(scale=1.0),
gamma_regularizer=tf.contrib.layers.l2_regularizer(scale=1.0))
logits = pf.dense(fc, num_class, 'logits', is_training, with_bn=False, activation=None)
probs = tf.nn.softmax(logits, name='probs')
labels_2d = tf.expand_dims(labels, axis=-1, name='labels_2d')
labels_tile = tf.tile(labels_2d, (1, tf.shape(probs)[1]), name='labels_tile')
#TODO
labels_tile = labels
logits = tf.reduce_mean(logits, axis=1)
#TODO
loss_op = tf.losses.sparse_softmax_cross_entropy(labels=labels_tile, logits=logits)
#loss_op = tf.nn.softmax_cross_entropy_with_logits(labels=labels_tile, logits=logits)
lr_exp_op = tf.train.exponential_decay(0.001, global_step, 1000,
0.5, staircase=True)
lr_clip_op = tf.maximum(lr_exp_op, 1e-6)
reg_loss = 0.0 * tf.losses.get_regularization_loss()
optimizer = tf.train.AdamOptimizer(learning_rate=lr_clip_op, epsilon=1e-2)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss_op + reg_loss, global_step=global_step)
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
gpu_options = tf.GPUOptions(allow_growth=True)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
sess.run(init_op)
pts_fts_v = [np.array([[[2,3,5,1,6,4,8]], [[2,6,7,2,4,7,4]]]).astype(np.float32), \
np.array([[[2,6,7,2,4,7,4]], [[2,3,5,1,6,4,8]]]).astype(np.float32)]
labels_v = [np.array([0,3]).astype(np.int32), np.array([3,0]).astype(np.int32)]
for idx in range(10000):
out_val = \
sess.run([dense, fc, train_op, loss_op],
feed_dict={
pts_fts: pts_fts_v[idx%2],
labels: labels_v[idx%2],
is_training: True,
})
print('dense')
print(out_val[0])
print('fc')
print(out_val[1])
print(out_val[-1])
def xconv(pts,
fts,
qrs,
tag,
N,
K,
D,
P,
C,
C_pts_fts,
is_training,
with_X_transformation,
depth_multiplier,
sorting_method=None,
with_global=False):
_, indices_dilated = pf.knn_indices_general(
qrs, pts, K * D, True
) # 带孔K*D近邻； (N, P, 3) + (N, P, 3) => (N, P, K*D, 2) N=batch_num; P=point_num; 最后一维两个数：batch_indice + point_indice
indices = indices_dilated[:, :, ::
D, :] # (N, P, K, 2) 隔D取样，增加receptive field
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(
pts, indices, name=tag + 'nn_pts'
) # (N, P, 3) + (N, P, K, 2) => (N, P, K, 3) indices维度比pts高，最后一维做索引，batch_indice->N, point_indice->P, 最后剩下(N, P, 3)里的3是坐标，补进去返回形状是 (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(
nn_pts, nn_pts_center,
name=tag + 'nn_pts_local') # (N, P, K, 3)，减去中心点的坐标系，得到局部坐标系
# Prepare features to be transformed
# 2个全连接层 对点的相对坐标系升维，(N, P, K, 3) => (N, P, K, C_pts_fts)
# 升维的目的是为了和前面的特征拼接 (ct.)
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
# 如果第一次没有feature，就只使用从局部得到的点特征，如果有之前的特征，就拼接二者(ct.)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input') # 拼接坐标特征和之前的特征
if with_X_transformation:
######################## X-transformation #########################
# (N, P, K, 3) > (N, P, 1, K * K) ； kernel size 是(1, K)
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
# (N, P, 1, K * K) > (N, P, K, K)
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
# (N, P, K, K) > (N, P, 1, K * K) 可能为了较少参数量用depthwise_conv2d换conv2d
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
# (N, P, 1, K * K) > (N, P, K, K)
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
# (N, P, K, K) > (N, P, 1, K * K)
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None)
# (N, P, 1, K * K) > (N, P, K, K) 最后的x矩阵 （K*K）
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
# 用得到的 x 矩阵乘以之前的特征，所以这里其实是有2个分支，一个计算x矩阵,另外一个计算特征；(ct.)
# (N, P, K, K) * (N, P, K, C_pts_fts) = (N, P, K, C_pts_fts)
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
# 最后的分离卷积
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier
) # (N, P, K, C_pts_fts) -> (N, P, 1, C)
fts_conv_3d = tf.squeeze(fts_conv, axis=2,
name=tag + 'fts_conv_3d') # (N, P, C)
# 用代表点全局位置信息,
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(
fts_global_0, C // 4, tag + 'fts_global', is_training
) # (N, P, C//4) 最后一层时作者使receptive field < 1让model更会捕捉local feature; 从“subvolume supervision”得到启发 to further address the over-fitting problem.;
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def __init__(self, points, features, num_class, is_training, setting, task):
xconv_params = setting.xconv_params
fc_params = setting.fc_params
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
if setting.sampling == 'fps':
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
C_fts = xconv_params[0]['C'] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
P = layer_param['P']
C = layer_param['C']
links = layer_param['links']
if setting.sampling != 'random' and links:
print('Error: flexible links are supported only when random sampling is used!')
exit()
# get k-nearest points
pts = self.layer_pts[-1]
fts = self.layer_fts[-1]
if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']):
qrs = self.layer_pts[-1]
else:
if setting.sampling == 'fps':
indices = tf_sampling.farthest_point_sample(P, pts)
qrs = tf_sampling.gather_point(pts, indices) # (N,P,3)
elif setting.sampling == 'ids':
indices = pf.inverse_density_sampling(pts, K, P)
qrs = tf.gather_nd(pts, indices)
elif setting.sampling == 'random':
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3)
else:
print('Unknown sampling method!')
exit()
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1]['C']
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation,
depth_multiplier, sorting_method, layer_idx != len(xconv_params) - 1)
fts_list = []
for link in links:
fts_from_link = self.layer_fts[link]
if fts_from_link is not None:
fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1),
name=tag + 'fts_slice_' + str(-link))
C_forward = math.ceil(fts_slice.get_shape().as_list()[-1] / (-link))
fts_forward = pf.dense(fts_slice, C_forward, tag + 'fts_forward_' + str(-link), is_training)
fts_list.append(fts_forward)
if fts_list:
fts_list.append(fts_xconv)
self.layer_fts.append(tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat'))
else:
self.layer_fts.append(fts_xconv)
if task == 'segmentation':
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
pts_layer_idx = layer_param['pts_layer_idx']
qrs_layer_idx = layer_param['qrs_layer_idx']
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
P = xconv_params[qrs_layer_idx]['P']
C = xconv_params[qrs_layer_idx]['C']
C_prev = xconv_params[pts_layer_idx]['C']
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation,
depth_multiplier, sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
C = layer_param['C']
dropout_rate = layer_param['dropout_rate']
fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training)
fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
if task == 'classification':
fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keep_dims=True, name='fc_mean')
self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean)
self.logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None)
self.probs = tf.nn.softmax(self.logits, name='probs')
def __init__(self, points, features, images, image_xy, num_class,
is_training, setting, task):
print("points:", points)
print("features:", features)
xconv_params = setting.xconv_params
print("xconv_params:", xconv_params)
fc_params = setting.fc_params
print("fc_params:", fc_params)
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
point_num = tf.shape(points)[1]
if setting.with_fps:
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
C_fts = xconv_params[0][-1] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K, D, P, C = layer_param
fts = self.layer_fts[-1]
# get k centroid points
pts = self.layer_pts[-1]
if P == -1:
qrs = self.layer_pts[-1]
else:
if setting.with_fps:
fps_indices = tf_sampling.farthest_point_sample(P, pts)
batch_indices = tf.tile(
tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1))
fps_indices_g = tf.concat(
[batch_indices,
tf.expand_dims(fps_indices, -1)],
axis=-1)
qrs = tf.gather_nd(pts, fps_indices_g,
name=tag + 'qrs') # (N, P, 3)
else:
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1][-1]
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method)
self.layer_fts.append(fts_xconv)
if task == 'segmentation':
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K, D, pts_layer_idx, qrs_layer_idx = layer_param
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[
pts_layer_idx +
1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
_, _, P, C = xconv_params[qrs_layer_idx]
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C,
C_pts_fts, is_training,
with_X_transformation, depth_multiplier,
sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs],
axis=-1,
name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse',
is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.fc_layers[-1], channel_num,
'fc{:d}'.format(layer_idx), is_training)
fc_drop = tf.layers.dropout(fc,
drop_rate,
training=is_training,
name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
if task == 'classification':
fc_mean = tf.reduce_mean(self.fc_layers[-1],
axis=1,
keep_dims=True,
name='fc_mean')
self.fc_layers[-1] = tf.cond(is_training,
lambda: self.fc_layers[-1],
lambda: fc_mean)
self.logits = pf.dense(self.fc_layers[-1],
num_class,
'logits',
is_training,
with_bn=False,
activation=None)
self.probs = tf.nn.softmax(self.logits, name='probs')
def deconv_new(pts,
fts,
qrs,
tag,
N,
K,
radius,
P,
C,
C_pts_fts,
is_training,
with_X_transformation,
depth_multiplier,
D=1,
sorting_method=None,
with_global=False,
knn=False):
"""
pts: points of previous layer, e.g.,(B, N/2, 3)
fts: point features of previous layer, e.g.,(B, N/2, C)
qrs: selected representative points of this layer, e.g.,(B, N, 3)
N: batch_size,
K: neighbor_size,
D: dilation parameter,
P: the number of selected representative points,
C: output feature number per point,
C_pts_fts: feature number for each local point
radius: float32, the radius of query ball search
knn: True: knn; False: query ball
"""
dist, idx = three_nn(qrs, pts)
dist = tf.maximum(dist, 1e-10)
norm = tf.reduce_sum((1.0 / dist), axis=2, keep_dims=True)
norm = tf.tile(norm, [1, 1, 3])
weight = (1.0 / dist) / norm
interpolated_fts = three_interpolate(fts, idx, weight) # (B, N, C)
if knn:
_, indices_dilated = pf.knn_indices_general(qrs, qrs, K * D, True)
indices = indices_dilated[:, :, ::D, :]
nn_pts = tf.gather_nd(qrs, indices,
name=tag + 'nn_pts') # (B, N, K, 3)
nn_fts_from_prev = tf.gather_nd(interpolated_fts,
indices,
name=tag + 'nn_fts')
else:
indices, pts_cnt = query_ball_point(radius, K, qrs, qrs)
nn_pts = group_point(qrs, indices)
nn_fts_from_prev = group_point(interpolated_fts, indices)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') # (B, N, 1, 3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + 'nn_pts_local') # (B, N, K, 3)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
if with_X_transformation:
######################## X-transformation #########################
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training,
(1, K)) # following paper
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training,
(1, K)) # following paper
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None) # following paper
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_',
is_training)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def xconv_cov(cov,
pts,
fts,
qrs,
tag,
N,
K,
D,
P,
C,
C_pts_fts,
is_training,
with_X_transformation,
depth_multiplier,
sorting_method=None,
with_global=False):
"""
cov: point covariance (B, N, 9) of previous layer
pts: points of previous layer,
fts: point features of previous layer,
qrs: selected representative points of this layer,
N: batch_size,
K: neighbor_size,
D: dilation parameter,
P: the number of selected representative points,
C: output feature number per point,
C_pts_fts: feature number for each local point
"""
if D == 1:
_, indices = pf.knn_indices_general(qrs, pts, K, True)
else:
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_covs = tf.gather_nd(cov, indices, name=tag + 'nn_pts') # (N, P, K, 9)
# nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3)
# nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_covs,
C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training,
with_bn=False) # following paper
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0,
C_pts_fts,
tag + 'nn_fts_from_pts',
is_training,
with_bn=False) # following paper
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
if with_X_transformation:
######################## X-transformation #########################
X_0 = pf.conv2d(nn_covs,
K * K,
tag + 'X_0',
is_training, (1, K),
with_bn=False) # following paper
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK,
K,
tag + 'X_1',
is_training, (1, K),
with_bn=False) # following paper
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
with_bn=False,
activation=None) # following paper
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier,
with_bn=True)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs,
C // 4,
tag + 'fts_global_0',
is_training,
with_bn=True)
fts_global = pf.dense(fts_global_0,
C // 4,
tag + 'fts_global_',
is_training,
with_bn=True)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
