def sample_and_group(npoint, radius, nsample, xyz, points, knn=False, use_xyz=True):
'''
Input:
npoint: int32
radius: float32
nsample: int32
xyz: (batch_size, ndataset, 3) TF tensor
points: (batch_size, ndataset, channel) TF tensor, if None will just use xyz as points
knn: bool, if True use kNN instead of radius search
use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features
Output:
new_xyz: (batch_size, npoint, 3) TF tensor
new_points: (batch_size, npoint, nsample, 3+channel) TF tensor
idx: (batch_size, npoint, nsample) TF tensor, indices of local points as in ndataset points
grouped_xyz: (batch_size, npoint, nsample, 3) TF tensor, normalized point XYZs
(subtracted by seed point XYZ) in local regions
'''
new_xyz = gather_point(xyz, farthest_point_sample(npoint, xyz)) # (batch_size, npoint, 3)
if knn:
_, idx = knn_point(nsample, xyz, new_xyz)
else:
idx, pts_cnt = query_ball_point(radius, nsample, xyz, new_xyz)
grouped_xyz = group_point(xyz, idx) # (batch_size, npoint, nsample, 3)
grouped_xyz -= tf.tile(tf.expand_dims(new_xyz, 2), [1, 1, nsample, 1]) # translation normalization
if points is not None:
grouped_points = group_point(points, idx) # (batch_size, npoint, nsample, channel)
if use_xyz:
new_points = tf.concat([grouped_xyz, grouped_points], axis=-1) # (batch_size, npoint, nample, 3+channel)
else:
new_points = grouped_points
else:
new_points = grouped_xyz
return new_xyz, new_points, idx, grouped_xyz
def __init__(self, points, features, num_class, is_training, setting,
task):
xconv_params = setting.xconv_params
fc_params = setting.fc_params
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
if setting.with_fps:
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
C_fts = xconv_params[0][-1] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K, D, P, C = layer_param
# get k-nearest points
pts = self.layer_pts[-1]
fts = self.layer_fts[-1]
if P == -1:
qrs = points
else:
if setting.with_fps:
qrs = tf_sampling.gather_point(
pts, tf_sampling.farthest_point_sample(P,
pts)) # (N,P,3)
else:
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1][-1]
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method)
self.layer_fts.append(fts_xconv)
if task == 'segmentation':
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K, D, pts_layer_idx, qrs_layer_idx = layer_param
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[
pts_layer_idx +
1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
_, _, P, C = xconv_params[qrs_layer_idx]
_, _, _, C_prev = xconv_params[pts_layer_idx]
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C,
C_pts_fts, is_training,
with_X_transformation, depth_multiplier,
sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs],
axis=-1,
name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse',
is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.fc_layers[-1], channel_num,
'fc{:d}'.format(layer_idx), is_training)
fc_drop = tf.layers.dropout(fc,
drop_rate,
training=is_training,
name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
logits = pf.dense(self.fc_layers[-1],
num_class,
'logits',
is_training,
with_bn=False,
activation=None)
if task == 'classification':
logits_mean = tf.reduce_mean(logits,
axis=1,
keep_dims=True,
name='logits_mean')
self.logits = tf.cond(is_training, lambda: logits,
lambda: logits_mean)
elif task == 'segmentation':
self.logits = logits
else:
print('Unknown task!')
exit()
self.probs = tf.nn.softmax(self.logits, name='probs')
def __init__(self, points, features, num_class, is_training, setting, task):
xconv_params = setting.xconv_params
fc_params = setting.fc_params
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
if setting.sampling == 'fps':
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
C_fts = xconv_params[0]['C'] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
P = layer_param['P']
C = layer_param['C']
links = layer_param['links']
if setting.sampling != 'random' and links:
print('Error: flexible links are supported only when random sampling is used!')
exit()
# get k-nearest points
pts = self.layer_pts[-1]
fts = self.layer_fts[-1]
if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']):
qrs = self.layer_pts[-1]
else:
if setting.sampling == 'fps':
indices = tf_sampling.farthest_point_sample(P, pts)
qrs = tf_sampling.gather_point(pts, indices) # (N,P,3)
elif setting.sampling == 'ids':
indices = pf.inverse_density_sampling(pts, K, P)
qrs = tf.gather_nd(pts, indices)
elif setting.sampling == 'random':
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3)
else:
print('Unknown sampling method!')
exit()
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1]['C']
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation,
depth_multiplier, sorting_method, layer_idx != len(xconv_params) - 1)
fts_list = []
for link in links:
fts_from_link = self.layer_fts[link]
if fts_from_link is not None:
fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1),
name=tag + 'fts_slice_' + str(-link))
C_forward = math.ceil(fts_slice.get_shape().as_list()[-1] / (-link))
fts_forward = pf.dense(fts_slice, C_forward, tag + 'fts_forward_' + str(-link), is_training)
fts_list.append(fts_forward)
if fts_list:
fts_list.append(fts_xconv)
self.layer_fts.append(tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat'))
else:
self.layer_fts.append(fts_xconv)
if task == 'segmentation':
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K = layer_param['K']
D = layer_param['D']
pts_layer_idx = layer_param['pts_layer_idx']
qrs_layer_idx = layer_param['qrs_layer_idx']
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
P = xconv_params[qrs_layer_idx]['P']
C = xconv_params[qrs_layer_idx]['C']
C_prev = xconv_params[pts_layer_idx]['C']
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation,
depth_multiplier, sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
C = layer_param['C']
dropout_rate = layer_param['dropout_rate']
fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training)
fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
if task == 'classification':
fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keep_dims=True, name='fc_mean')
self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean)
self.logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None)
self.probs = tf.nn.softmax(self.logits, name='probs')
ops.NoGradient('AuctionMatch')
# TF1.0 API requires set shape in C++
# @tf.RegisterShape('AuctionMatch')
# def _auction_match_shape(op):
# shape1=op.inputs[0].get_shape().with_rank(3)
# shape2=op.inputs[1].get_shape().with_rank(3)
# return [
# tf.TensorShape([shape1.dims[0],shape1.dims[1]]),
# tf.TensorShape([shape2.dims[0],shape2.dims[1]])
# ]
if __name__=='__main__':
from grouping import tf_grouping
from sampling import tf_sampling
npoint=4096
xyz1_in=tf.placeholder(tf.float32,shape=(32,npoint,3))
xyz2_in=tf.placeholder(tf.float32,shape=(32,npoint,3))
matchl_out,matchr_out=auction_match(xyz1_in,xyz2_in)
matched_out=tf_sampling.gather_point(xyz2_in,matchl_out)
import numpy as np
np.random.seed(100)
xyz1=np.random.randn(32,npoint,3).astype('float32')
xyz2=xyz1.copy()+np.random.randn(32,npoint,3)*0.01
for i in range(len(xyz2)):
xyz2[i]=np.roll(xyz2[i],i,axis=0)
with tf.Session('') as sess:
ret=sess.run(matched_out,feed_dict={xyz1_in:xyz1,xyz2_in:xyz2})
print(((xyz1-ret)**2).mean())
def __init__(self, points, features, images, image_xy, num_class,
is_training, setting, task):
print("points:", points)
print("features:", features)
xconv_params = setting.xconv_params
print("xconv_params:", xconv_params)
fc_params = setting.fc_params
print("fc_params:", fc_params)
# 从配置文件中获取后续分支的参数
extra_branch_xconv_params = setting.extra_branch_xconv_params
extra_branch_fc_params = setting.extra_branch_fc_params
mask_sample_num = setting.mask_sample_num
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
point_num = tf.shape(points)[1]
if setting.with_fps:
from sampling import tf_sampling
self.layer_pts = [points]
if features is None:
self.layer_fts = [features]
else:
C_fts = xconv_params[0][-1] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K, D, P, C = layer_param
fts = self.layer_fts[-1]
# get k centroid points
pts = self.layer_pts[-1]
if P == -1:
qrs = self.layer_pts[-1]
else:
if setting.with_fps:
fps_indices = tf_sampling.farthest_point_sample(P, pts)
batch_indices = tf.tile(
tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1))
fps_indices_g = tf.concat(
[batch_indices,
tf.expand_dims(fps_indices, -1)],
axis=-1)
qrs = tf.gather_nd(pts, fps_indices_g,
name=tag + 'qrs') # (N, P, 3)
else:
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1][-1]
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method, False)
self.layer_fts.append(fts_xconv)
if task == 'segmentation':
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K, D, pts_layer_idx, qrs_layer_idx = layer_param
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[
pts_layer_idx +
1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
_, _, P, C = xconv_params[qrs_layer_idx]
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C,
C_pts_fts, is_training,
with_X_transformation, depth_multiplier,
sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs],
axis=-1,
name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse',
is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
self.fc_layers = [self.layer_fts[-1]]
for layer_idx, layer_param in enumerate(fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.fc_layers[-1], channel_num,
'fc{:d}'.format(layer_idx), is_training)
fc_drop = tf.layers.dropout(fc,
drop_rate,
training=is_training,
name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
if task == 'classification':
fc_mean = tf.reduce_mean(self.fc_layers[-1],
axis=1,
keep_dims=True,
name='fc_mean')
self.fc_layers[-1] = tf.cond(is_training,
lambda: self.fc_layers[-1],
lambda: fc_mean)
self.logits = pf.dense(self.fc_layers[-1],
num_class,
'logits',
is_training,
with_bn=False,
activation=None)
self.probs = tf.nn.softmax(self.logits, name='probs')
# 从分割结果获取Mask点云(车辆点云)
probs_others = tf.slice(self.probs, [0, 0, 0], [N, point_num, 1])
probs_instance = tf.slice(self.probs, [0, 0, 1], [N, point_num, 1])
hold_sub_mask = tf.subtract(probs_instance,
probs_others,
name="hold_sub_mask")
indices_mask = tf.py_func(pf.instance_choice,
[hold_sub_mask, mask_sample_num], tf.int32)
indices_mask.set_shape(
[hold_sub_mask.get_shape()[0], mask_sample_num, 2])
points_mask = tf.gather_nd(points, indices_mask, name='points_mask')
# 计算mask点云重心
if setting.with_xyz_res:
points_mask_mean = tf.reduce_mean(points_mask,
axis=-2,
keep_dims=True,
name='points_mask_mean')
points_mask_mean_sq = tf.squeeze(points_mask_mean,
name='points_mask_mean_sq')
points_mask_centroid = tf.subtract(points_mask,
points_mask_mean,
name='points_mask_centroid')
eb_pts = points_mask_centroid
else:
eb_pts = points_mask
# 获取mask点云特征
if features is None:
features_mask_hd = None
else:
C_fts = extra_branch_xconv_params[0][-1] // 2
features_mask = tf.gather_nd(features,
indices_mask,
name='features_mask')
mask_features_hd = pf.dense(features_mask, C_fts,
'mask_features_hd', is_training)
eb_fts = mask_features_hd
# Bbox分支(尺寸与位置分支)
self.bbox_layer_fts = [eb_fts]
self.bbox_layer_pts = [eb_pts]
for layer_idx, layer_param in enumerate(extra_branch_xconv_params):
tag = 'bbox_branch_xconv_' + str(layer_idx + 1) + '_'
K, D, P, C = layer_param
pts = self.bbox_layer_pts[-1]
fts = self.bbox_layer_fts[-1]
# get centroid points
if P == -1:
qrs = eb_pts
else:
if setting.eb_with_fps:
fps_indices = tf_sampling.farthest_point_sample(P, pts)
batch_indices = tf.tile(
tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1))
fps_indices_g = tf.concat(
[batch_indices,
tf.expand_dims(fps_indices, -1)],
axis=-1)
qrs = tf.gather_nd(pts, fps_indices_g,
name=tag + 'qrs') # (N, P, 3)
else:
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
self.bbox_layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1][-1]
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method, False)
self.bbox_layer_fts.append(fts_xconv)
# HWL 预测分支 以用maxpooling特征进行预测为例
self.hwl_fc_layers = [
tf.reduce_max(self.bbox_layer_fts[-1], axis=1, name='hwl_fts_max')
]
for layer_idx, layer_param in enumerate(extra_branch_fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.hwl_fc_layers[-1],
channel_num,
'hwl_fc{:d}'.format(layer_idx),
is_training,
with_bn=setting.hwl_branch_setting["with_bn"])
fc_drop = tf.layers.dropout(
fc,
drop_rate,
training=is_training,
name='hwl_fc{:d}_drop'.format(layer_idx))
self.hwl_fc_layers.append(fc_drop)
self.logits_hwl = pf.dense(self.hwl_fc_layers[-1],
3,
'logits_hwl',
is_training,
with_bn=False,
activation=None)
# XYZ 预测分支 以用maxpooling特征进行预测为例
self.xyz_fc_layers = [
tf.reduce_max(self.bbox_layer_fts[-1], axis=1, name='xyz_fts_max')
]
for layer_idx, layer_param in enumerate(extra_branch_fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.xyz_fc_layers[-1],
channel_num,
'xyz_fc{:d}'.format(layer_idx),
is_training,
with_bn=setting.xyz_branch_setting["with_bn"])
fc_drop = tf.layers.dropout(
fc,
drop_rate,
training=is_training,
name='xyz_fc{:d}_drop'.format(layer_idx))
self.xyz_fc_layers.append(fc_drop)
if setting.with_xyz_res:
logits_xyz_res = pf.dense(self.xyz_fc_layers[-1],
3,
'logits_xyz_res',
is_training,
with_bn=False,
activation=None)
self.logits_xyz = tf.add(points_mask_mean_sq,
logits_xyz_res,
name='logits_xyz')
else:
self.logits_xyz = pf.dense(self.xyz_fc_layers[-1],
3,
'logits_xyz_res',
is_training,
with_bn=False,
activation=None)
# Ry分支(旋转角分支)
self.ry_layer_fts = [eb_fts]
self.ry_layer_pts = [eb_pts]
for layer_idx, layer_param in enumerate(extra_branch_xconv_params):
tag = 'ry_branch_xconv_' + str(layer_idx + 1) + '_'
K, D, P, C = layer_param
fts = self.ry_layer_fts[-1]
# get centroid points
pts = self.ry_layer_pts[-1]
if P == -1:
qrs = eb_pts
else:
if setting.eb_with_fps:
fps_indices = tf_sampling.farthest_point_sample(P, pts)
qrs = tf_sampling.gather_point(pts, fps_indices) # (N,P,3)
else:
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1),
name=tag + 'qrs') # (N, P, 3)
self.ry_layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1][-1]
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts,
is_training, with_X_transformation,
depth_multiplier, sorting_method, False)
self.ry_layer_fts.append(fts_xconv)
# Ry 分类预测分支 以用maxpooling特征进行预测为例
self.ry_fc_layers = [
tf.reduce_max(self.ry_layer_fts[-1], axis=1, name='ry_fts_max')
]
for layer_idx, layer_param in enumerate(extra_branch_fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.ry_fc_layers[-1],
channel_num,
'ry_fc{:d}'.format(layer_idx),
is_training,
with_bn=setting.ry_branch_setting["with_bn"])
fc_drop = tf.layers.dropout(
fc,
drop_rate,
training=is_training,
name='ry_fc{:d}_drop'.format(layer_idx))
self.ry_fc_layers.append(fc_drop)
self.logits_ry = pf.dense(self.ry_fc_layers[-1],
24,
'logits_ry',
is_training,
with_bn=False,
activation=None)
self.probs_ry = tf.nn.softmax(self.logits_ry, name='probs_ry')