def local_map_loss(self, temp_points, deform_points, K=8, D=1):
if D == 1:
_, indices = pf.knn_indices_general(temp_points, temp_points, K,
False)
else:
_, indices_dilated = pf.knn_indices_general(
temp_points, temp_points, K * D, True)
indices = indices_dilated[:, :, ::D, :]
batch_size = deform_points.get_shape()[0].value
points_num = deform_points.get_shape()[1].value
offsets = tf.subtract(deform_points, temp_points)
dists = tf.sqrt(tf.reduce_sum(tf.pow(offsets, 2), axis=-1))
directions = offsets / tf.tile(tf.expand_dims(dists, axis=-1),
[1, 1, 3])
nn_directions = tf.gather_nd(directions, indices,
name='nn_pts') # batch*point*k*3
nn_cosin_matrix = 1 - tf.matmul(
nn_directions, nn_directions, transpose_b=True)
nn_cosin_matrix = tf.reshape(nn_cosin_matrix,
shape=[batch_size, points_num, K * K])
loss = tf.reduce_mean(nn_cosin_matrix, axis=[0, 1, 2])
loss = loss + 0.1 * tf.reduce_mean(dists, axis=[0, 1])
return loss
def points_laplace_coord(points, K=8, D=1):
if D == 1:
_, indices = pf.knn_indices_general(points, points, K, False)
else:
_, indices_dilated = pf.knn_indices_general(points, points, K*D, True)
indices = indices_dilated[:, :, ::D, :]
# calculate laplacian coordinate:
nn_points = tf.gather_nd(points, indices, name = 'nn_pts')
points_tile = tf.expand_dims(points, axis=2, name = 'points_expand')
points_tile = tf.tile(points_tile, [1, 1, K, 1])
mean_coords = tf.reduce_mean(tf.subtract(points_tile, nn_points), axis=-2)
lap_coords = tf.subtract(points, mean_coords)
return lap_coords
def shellconv(pts, fts_prev, qrs, is_training, tag, K, D, P, C, with_local, bn_decay=None):
indices = pf.knn_indices_general(qrs, pts, K, True)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3)
[N,P,K,dim] = nn_pts_local.shape # (N, P, K, 3)
nn_fts_local = None
C_pts_fts = 64
if with_local:
nn_fts_local = dense(nn_pts_local, C_pts_fts // 2, is_training, tag + 'nn_fts_from_pts_0', bn_decay=bn_decay)
nn_fts_local = dense(nn_fts_local, C_pts_fts, is_training, tag + 'nn_fts_from_pts', bn_decay=bn_decay)
else:
nn_fts_local = nn_pts_local
if fts_prev is not None:
fts_prev = tf.gather_nd(fts_prev, indices, name=tag + 'fts_prev') # (N, P, K, 3)
pts_X_0 = tf.concat([nn_fts_local,fts_prev], axis=-1)
else:
pts_X_0 = nn_fts_local
s = int(K.value/D) # no. of divisions
feat_max = tf.layers.max_pooling2d(pts_X_0, [1,s], strides=[1,s], padding='valid', name=tag+'maxpool_0')
fts_X = conv2d(feat_max, C, name=tag+'conv', is_training=is_training, kernel_size=[1,feat_max.shape[-2].value])
fts_X = tf.squeeze(fts_X, axis=-2)
return fts_X
def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier,
sorting_method=None, with_global=False):
if D == 1:
_, indices = pf.knn_indices_general(qrs, pts, K, True)
else:
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input')
if with_X_transformation:
######################## X-transformation #########################
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None)
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training)
return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def point_conv(pts,
fts,
qrs,
K,
D,
C,
is_training,
activation=tf.nn.relu,
bn=True,
sorting_method='cxyz',
center_patch=True,
tag='point_conv'):
'''
pts: xyz points, Tensor, (batch_size, npoint, 3)
fts: features, Tensor, (batch_size, npoint, channel_fts)
qrs: xyz points as queries, Tensor, (batch_size, n_qrs, 3)
K: number of neighbors for one query, scalar, int
D: dilation rate
C: output feature channel number
'''
with tf.variable_scope(tag):
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(pts, indices, name='nn_pts') # (N, P, K, 3)
if center_patch:
nn_pts_center = tf.expand_dims(
qrs, axis=2, name='nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(
nn_pts, nn_pts_center,
name='nn_pts_local') # (N, P, K, 3), move to local frame
else:
nn_pts_local = nn_pts
if fts is None:
nn_fts_input = nn_pts_local
else:
nn_fts_from_prev = tf.gather_nd(fts, indices, 'nn_fts_from_prev')
nn_fts_input = tf.concat(
[nn_pts_local, nn_fts_from_prev], axis=-1,
name='nn_fts_input') # (N, P, K, 3+C_prev_fts)
fts_conv = pf.conv2d(nn_fts_input,
C,
'fts_conv',
is_training, (1, K),
with_bn=bn,
activation=activation) # (N, P, 1, C)
fts_conv_3d = tf.squeeze(fts_conv, axis=2,
name='fts_conv_3d') # (N, P, C)
return fts_conv_3d, indices
def sa_conv(pts, fts, qrs, K, D, mlp, bn, is_training, bn_decay, scope):
"""
pts: points of previous layer,
fts: point features of previous layer,
qrs: selected representative points of this layer,
N: batch_size,
K: neighbor_size,
D: dilation parameter,
P: the number of selected representative points,
C: output feature number per point,
C_pts_fts: feature number for each local point
"""
if D == 1:
_, indices = pf.knn_indices_general(qrs, pts, K, True)
else:
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
new_points = tf.gather_nd(fts, indices) # (N, P, K, 3)
with tf.variable_scope(scope) as sc:
for i, num_out_channel in enumerate(mlp):
new_points = tf_util.conv2d(new_points,
num_out_channel, [1, 1],
padding='VALID',
stride=[1, 1],
bn=bn,
is_training=is_training,
scope='conv%d' % (i),
bn_decay=bn_decay)
new_points = tf.reduce_max(new_points,
axis=[2],
keep_dims=True,
name='maxpool')
fts_conv_3d = tf.squeeze(new_points,
[2]) # (batch_size, npoints, mlp[-1])
return fts_conv_3d
def shellconv(pts, fts_prev, qrs, is_training, tag, K, D, P, C, with_local, bn_decay=None, norm='l2'):
indices = pf.knn_indices_general(qrs, pts, K, True, norm=norm)
if norm == 'l1':
# Adjust the max distance so that the hypercube is of the same volume
# as the hyperball, for fairness.
#
# Oh, wait, shell size D is the number of points, not the max distance...
# D = int(D * np.sqrt(3.141592) / np.exp(1.0 / C * sp.special.loggamma(0.5 * C + 1)))
pass
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3)
[N,P,K,dim] = nn_pts_local.shape # (N, P, K, 3)
nn_fts_local = None
C_pts_fts = 64
if with_local:
nn_fts_local = dense(nn_pts_local, C_pts_fts // 2, is_training, tag + 'nn_fts_from_pts_0',bn_decay=bn_decay)
nn_fts_local = dense(nn_fts_local, C_pts_fts, is_training, tag + 'nn_fts_from_pts',bn_decay=bn_decay)
else:
nn_fts_local = nn_pts_local
if fts_prev is not None:
fts_prev = tf.gather_nd(fts_prev, indices, name=tag + 'fts_prev') # (N, P, K, 3)
pts_X_0 = tf.concat([nn_fts_local,fts_prev], axis=-1)
else:
pts_X_0 = nn_fts_local
s = int(K.value/D) # no. of divisions
feat_max = tf.layers.max_pooling2d(pts_X_0, [1,s], strides=[1,s], padding='valid', name=tag+'maxpool_0')
fts_X = conv2d(feat_max, C, name=tag+'conv', is_training=is_training, kernel_size=[1,feat_max.shape[-2].value])
fts_X = tf.squeeze(fts_X, axis=-2)
return fts_X
def xconv(pts,
fts,
qrs,
tag,
N,
K,
D,
P,
C,
depth_multiplier,
is_training,
C_pts_fts,
with_x_transformation=True,
sorting_method=None,
with_global=False):
###xconv(pts,fts,qrs,tag,N,K,D,P,C,C_pts_fts,is_training,with_X_transformation,depth_multipiler,sorting_method,with_global)
# @params:
# N: Number of output points
# K: Number of nearest neighbot
# D: dilation rate
# C: Number of output channels
# P:the representative point number in the output, -1 means all input points are output representative points
# x_transformation: replace max_pooling in PointNet
# sorting_method:
# with_global
# pts:Input point cloud
# qrs:queries
# fts:features
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') #(N,P,K,3)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') #(N,P,1,3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + 'nn_pts_local') #(N,P,K,3)
#Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
#X_transformation
if with_x_transformation:
############################X_transformation#######################################################
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None)
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
#####################################################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global',
is_training)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def RIConv(pts, fts_prev, qrs, is_training, tag, K, D, P, C, with_local, bn_decay=None):
indices = pf.knn_indices_general(qrs, pts, int(K), True)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts')
nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center')
nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local')
dists_local = tf.norm(nn_pts_local, axis=-1, keepdims=True) # dist to center
mean_local = tf.reduce_mean(nn_pts, axis=-2, keepdims=True)
mean_global = tf.reduce_mean(pts, axis=-2, keepdims=True)
mean_global = tf.expand_dims(mean_global, axis=-2)
nn_pts_local_mean = tf.subtract(nn_pts, mean_local, name=tag + 'nn_pts_local_mean')
dists_local_mean = tf.norm(nn_pts_local_mean, axis=-1, keepdims=True) # dist to local mean
vec = mean_local - nn_pts_center
vec_dist = tf.norm(vec, axis=-1, keepdims =True)
vec_norm = tf.divide(vec, vec_dist)
vec_norm = tf.where(tf.is_nan(vec_norm), tf.ones_like(vec_norm) * 0, vec_norm)
nn_pts_local_proj = tf.matmul(nn_pts_local, vec_norm, transpose_b=True)
nn_pts_local_proj_dot = tf.divide(nn_pts_local_proj, dists_local)
nn_pts_local_proj_dot = tf.where(tf.is_nan(nn_pts_local_proj_dot), tf.ones_like(nn_pts_local_proj_dot) * 0, nn_pts_local_proj_dot) # check nan
nn_pts_local_proj_2 = tf.matmul(nn_pts_local_mean, vec_norm, transpose_b=True)
nn_pts_local_proj_dot_2 = tf.divide(nn_pts_local_proj_2, dists_local_mean)
nn_pts_local_proj_dot_2 = tf.where(tf.is_nan(nn_pts_local_proj_dot_2), tf.ones_like(nn_pts_local_proj_dot_2) * 0, nn_pts_local_proj_dot_2) # check nan
nn_fts = tf.concat([dists_local, dists_local_mean, nn_pts_local_proj_dot, nn_pts_local_proj_dot_2], axis=-1) # d0 d1 a0 a1
# compute indices from nn_pts_local_proj
vec = mean_global - nn_pts_center
vec_dist = tf.norm(vec, axis=-1, keepdims =True)
vec_norm = tf.divide(vec, vec_dist)
nn_pts_local_proj = tf.matmul(nn_pts_local, vec_norm, transpose_b=True)
proj_min = tf.reduce_min(nn_pts_local_proj, axis=-2, keepdims=True)
proj_max = tf.reduce_max(nn_pts_local_proj, axis=-2, keepdims=True)
seg = (proj_max - proj_min) / D
vec_tmp = tf.range(0, D, 1, dtype=tf.float32)
vec_tmp = tf.reshape(vec_tmp, (1,1,1,D))
limit_bottom = vec_tmp * seg + proj_min
limit_up = limit_bottom + seg
idx_up = nn_pts_local_proj <= limit_up
idx_bottom = nn_pts_local_proj >= limit_bottom
idx = tf.to_float(tf.equal(idx_bottom, idx_up))
idx_expand = tf.expand_dims(idx, axis=-1)
[N,P,K,dim] = nn_fts.shape # (N, P, K, 3)
nn_fts_local = None
if with_local:
C_pts_fts = 64
nn_fts_local_reshape = tf.reshape(nn_fts, (-1,P*K,dim,1))
nn_fts_local = tf_util.conv2d(nn_fts_local_reshape, C_pts_fts//2, [1,dim],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope=tag+'conv_pts_fts_0', bn_decay=bn_decay)
nn_fts_local = tf_util.conv2d(nn_fts_local, C_pts_fts, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope=tag+'conv_pts_fts_1', bn_decay=bn_decay)
nn_fts_local = tf.reshape(nn_fts_local, (-1,P,K,C_pts_fts))
else:
nn_fts_local = nn_fts
if fts_prev is not None:
fts_prev = tf.gather_nd(fts_prev, indices, name=tag + 'fts_prev') # (N, P, K, 3)
pts_X_0 = tf.concat([nn_fts_local,fts_prev], axis=-1)
else:
pts_X_0 = nn_fts_local
pts_X_0_expand = tf.expand_dims(pts_X_0, axis=-2)
nn_fts_rect = pts_X_0_expand * idx_expand
idx = tf.to_float(nn_fts_rect == 0.0)
nn_fts_rect = nn_fts_rect + idx*(-99999999999.0)
nn_fts_rect = tf.reduce_max(nn_fts_rect, axis=-3)
# nn_fts_rect = tf.matmul(idx_mean, pts_X_0, transpose_a = True)
fts_X = tf_util.conv2d(nn_fts_rect, C, [1,nn_fts_rect.shape[-2].value],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope=tag+'conv', bn_decay=bn_decay)
return tf.squeeze(fts_X, axis=-2)
def xconv(pts,
fts,
qrs,
tag,
N,
K,
D,
P,
C,
C_pts_fts,
is_training,
with_X_transformation,
depth_multiplier,
sorting_method=None,
with_global=False):
_, indices_dilated = pf.knn_indices_general(
qrs, pts, K * D, True
) # 带孔K*D近邻； (N, P, 3) + (N, P, 3) => (N, P, K*D, 2) N=batch_num; P=point_num; 最后一维两个数：batch_indice + point_indice
indices = indices_dilated[:, :, ::
D, :] # (N, P, K, 2) 隔D取样，增加receptive field
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(
pts, indices, name=tag + 'nn_pts'
) # (N, P, 3) + (N, P, K, 2) => (N, P, K, 3) indices维度比pts高，最后一维做索引，batch_indice->N, point_indice->P, 最后剩下(N, P, 3)里的3是坐标，补进去返回形状是 (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(
nn_pts, nn_pts_center,
name=tag + 'nn_pts_local') # (N, P, K, 3)，减去中心点的坐标系，得到局部坐标系
# Prepare features to be transformed
# 2个全连接层 对点的相对坐标系升维，(N, P, K, 3) => (N, P, K, C_pts_fts)
# 升维的目的是为了和前面的特征拼接 (ct.)
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
# 如果第一次没有feature，就只使用从局部得到的点特征，如果有之前的特征，就拼接二者(ct.)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input') # 拼接坐标特征和之前的特征
if with_X_transformation:
######################## X-transformation #########################
# (N, P, K, 3) > (N, P, 1, K * K) ； kernel size 是(1, K)
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
# (N, P, 1, K * K) > (N, P, K, K)
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
# (N, P, K, K) > (N, P, 1, K * K) 可能为了较少参数量用depthwise_conv2d换conv2d
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
# (N, P, 1, K * K) > (N, P, K, K)
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
# (N, P, K, K) > (N, P, 1, K * K)
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None)
# (N, P, 1, K * K) > (N, P, K, K) 最后的x矩阵 （K*K）
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
# 用得到的 x 矩阵乘以之前的特征，所以这里其实是有2个分支，一个计算x矩阵,另外一个计算特征；(ct.)
# (N, P, K, K) * (N, P, K, C_pts_fts) = (N, P, K, C_pts_fts)
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
# 最后的分离卷积
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier
) # (N, P, K, C_pts_fts) -> (N, P, 1, C)
fts_conv_3d = tf.squeeze(fts_conv, axis=2,
name=tag + 'fts_conv_3d') # (N, P, C)
# 用代表点全局位置信息,
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(
fts_global_0, C // 4, tag + 'fts_global', is_training
) # (N, P, C//4) 最后一层时作者使receptive field < 1让model更会捕捉local feature; 从“subvolume supervision”得到启发 to further address the over-fitting problem.;
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def xconv_cov(cov,
pts,
fts,
qrs,
tag,
N,
K,
D,
P,
C,
C_pts_fts,
is_training,
with_X_transformation,
depth_multiplier,
sorting_method=None,
with_global=False):
"""
cov: point covariance (B, N, 9) of previous layer
pts: points of previous layer,
fts: point features of previous layer,
qrs: selected representative points of this layer,
N: batch_size,
K: neighbor_size,
D: dilation parameter,
P: the number of selected representative points,
C: output feature number per point,
C_pts_fts: feature number for each local point
"""
if D == 1:
_, indices = pf.knn_indices_general(qrs, pts, K, True)
else:
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_covs = tf.gather_nd(cov, indices, name=tag + 'nn_pts') # (N, P, K, 9)
# nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3)
# nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_covs,
C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training,
with_bn=False) # following paper
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0,
C_pts_fts,
tag + 'nn_fts_from_pts',
is_training,
with_bn=False) # following paper
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
if with_X_transformation:
######################## X-transformation #########################
X_0 = pf.conv2d(nn_covs,
K * K,
tag + 'X_0',
is_training, (1, K),
with_bn=False) # following paper
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK,
K,
tag + 'X_1',
is_training, (1, K),
with_bn=False) # following paper
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
with_bn=False,
activation=None) # following paper
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier,
with_bn=True)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs,
C // 4,
tag + 'fts_global_0',
is_training,
with_bn=True)
fts_global = pf.dense(fts_global_0,
C // 4,
tag + 'fts_global_',
is_training,
with_bn=True)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def deconv_new(pts,
fts,
qrs,
tag,
N,
K,
radius,
P,
C,
C_pts_fts,
is_training,
with_X_transformation,
depth_multiplier,
D=1,
sorting_method=None,
with_global=False,
knn=False):
"""
pts: points of previous layer, e.g.,(B, N/2, 3)
fts: point features of previous layer, e.g.,(B, N/2, C)
qrs: selected representative points of this layer, e.g.,(B, N, 3)
N: batch_size,
K: neighbor_size,
D: dilation parameter,
P: the number of selected representative points,
C: output feature number per point,
C_pts_fts: feature number for each local point
radius: float32, the radius of query ball search
knn: True: knn; False: query ball
"""
dist, idx = three_nn(qrs, pts)
dist = tf.maximum(dist, 1e-10)
norm = tf.reduce_sum((1.0 / dist), axis=2, keep_dims=True)
norm = tf.tile(norm, [1, 1, 3])
weight = (1.0 / dist) / norm
interpolated_fts = three_interpolate(fts, idx, weight) # (B, N, C)
if knn:
_, indices_dilated = pf.knn_indices_general(qrs, qrs, K * D, True)
indices = indices_dilated[:, :, ::D, :]
nn_pts = tf.gather_nd(qrs, indices,
name=tag + 'nn_pts') # (B, N, K, 3)
nn_fts_from_prev = tf.gather_nd(interpolated_fts,
indices,
name=tag + 'nn_fts')
else:
indices, pts_cnt = query_ball_point(radius, K, qrs, qrs)
nn_pts = group_point(qrs, indices)
nn_fts_from_prev = group_point(interpolated_fts, indices)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') # (B, N, 1, 3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + 'nn_pts_local') # (B, N, K, 3)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
if with_X_transformation:
######################## X-transformation #########################
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training,
(1, K)) # following paper
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training,
(1, K)) # following paper
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None) # following paper
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_',
is_training)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def point_conv_transpose(pts_in,
fts_in,
pts_out,
fts_out_skipped,
indices,
K,
D,
C,
is_training,
activation=tf.nn.relu,
bn=True,
sorting_method='cxyz',
center_patch=True,
tconv='fc',
tag='point_deconv'):
'''
pts_in: xyz points, Tensor, (batch_size, npoint, 3)
fts_in: features, Tensor, (batch_size, npoint, channel_fts)
pts_out: xyz points, Tensor, (batch_size, n_pts_out, 3), NOTE: pts_out is denser than pts.
K: number of neighbors for one query, scalar, int
D: dilation rate
C: output feature channel number
tconv: fc or t_conv
in the comments:
N - batch size
P - number of pts_in
K - number of neighbors
'''
with tf.variable_scope(tag):
batch_size = pts_out.shape[0]
n_pts_out = pts_out.shape[1]
n_pts_in = pts_in.shape[1]
if indices is None:
_, indices_dilated = pf.knn_indices_general(
pts_in, pts_out, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts_out, indices,
sorting_method) # (N, P, K, 2)
# move the gathered pts_out to local frame or not
nn_pts_out = tf.gather_nd(pts_out, indices,
name='nn_pts_out') # (N, P, K, 3)
if center_patch:
nn_pts_out_center = tf.expand_dims(
pts_in, axis=2, name='nn_pts_out_center') # (N, P, 1, 3)
nn_pts_out_local = tf.subtract(
nn_pts_out, nn_pts_out_center,
name='nn_pts_out_local') # (N, P, K, 3)
else:
nn_pts_out_local = nn_pts_out
if fts_in is None:
nn_fts_input = pts_in # (N, P, 3)
else:
# NOTE: do not concat pts_in with fts_in now, since fts_in already contains information of pts_in
nn_fts_input = fts_in # (N, P, C_fts_in)
nn_fts_input = tf.expand_dims(nn_fts_input,
axis=2) # (N, P, 1, C_fts_in)
# using fc
if tconv == 'fc':
fts_fc = pf.dense(nn_fts_input, K * C, 'fc_fts_prop',
is_training) # (N, P, 1, K*C)
fts_fc = tf.reshape(fts_fc,
(batch_size, n_pts_in, K, C)) # (N, P, K, C)
elif tconv == 't_conv':
fts_fc = pf.conv2d_transpose(nn_fts_input,
C,
tag + '_fts_deconv',
is_training,
kernel_size=(1, K),
strides=(1, 1),
with_bn=bn,
activation=activation) # (N, P, K, C)
if fts_out_skipped is not None:
nn_fts_out_skipped = tf.gather_nd(fts_out_skipped,
indices,
name='nn_fts_out_from_skip')
fts_all_concat = tf.concat(
[fts_fc, nn_fts_out_skipped, nn_pts_out_local],
axis=-1) # (N, P, K, C+3+channel_skipped)
else:
fts_all_concat = tf.concat([fts_fc, nn_pts_out_local],
axis=-1) # (N, P, K, C+3)
fts_pts_conv = pf.conv2d(
fts_all_concat,
C,
'fts_pts_conv1D',
is_training,
kernel_size=(1, 1),
strides=(1, 1),
with_bn=bn,
activation=activation) # (N, P, K, C+3+[ch_skip]) -> (N, P, K, C)
# summing up feature for each point
fts_dense = tf.scatter_nd(
tf.reshape(indices, (-1, 2)), tf.reshape(fts_pts_conv, (-1, C)),
(batch_size, n_pts_out, C)) # (N, N_PTS_OUT, c)
return fts_dense
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--filelist',
'-t',
help='Path to training set ground truth (.txt)',
required=True)
parser.add_argument('--filelist_val',
'-v',
help='Path to validation set ground truth (.txt)',
required=True)
parser.add_argument('--filelist_unseen',
'-u',
help='Path to validation set ground truth (.txt)',
required=True)
parser.add_argument('--load_ckpt',
'-l',
help='Path to a check point file for load')
parser.add_argument(
'--save_folder',
'-s',
help='Path to folder for saving check points and summary',
required=True)
parser.add_argument('--model', '-m', help='Model to use', required=True)
parser.add_argument('--setting',
'-x',
help='Setting to use',
required=True)
args = parser.parse_args()
time_string = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
root_folder = os.path.join(
args.save_folder,
'%s_%s_%s_%d' % (args.model, args.setting, time_string, os.getpid()))
if not os.path.exists(root_folder):
os.makedirs(root_folder)
print('PID:', os.getpid())
print(args)
model = importlib.import_module(args.model)
setting_path = os.path.join(os.path.dirname(__file__), args.model)
sys.path.append(setting_path)
setting = importlib.import_module(args.setting)
num_epochs = setting.num_epochs
batch_size = setting.batch_size
sample_num = setting.sample_num
step_val = setting.step_val
label_weights_list = setting.label_weights
label_weights_val = [1.0] * 1 + [1.0] * (setting.num_class - 1)
rotation_range = setting.rotation_range
rotation_range_val = setting.rotation_range_val
scaling_range = setting.scaling_range
scaling_range_val = setting.scaling_range_val
jitter = setting.jitter
jitter_val = setting.jitter_val
# Prepare inputs
print('{}-Preparing datasets...'.format(datetime.now()))
is_list_of_h5_list = not data_utils.is_h5_list(args.filelist)
if is_list_of_h5_list:
seg_list = data_utils.load_seg_list(args.filelist)
seg_list_idx = 0
filelist_train = seg_list[seg_list_idx]
seg_list_idx = seg_list_idx + 1
else:
filelist_train = args.filelist
data_train, _, data_num_train, label_train, _ = data_utils.load_seg(
filelist_train)
data_val, _, data_num_val, label_val, _ = data_utils.load_seg(
args.filelist_val)
data_unseen, _, data_num_unseen, label_unseen, _ = data_utils.load_seg(
args.filelist_unseen)
# shuffle
data_unseen, data_num_unseen, label_unseen = \
data_utils.grouped_shuffle([data_unseen, data_num_unseen, label_unseen])
# shuffle
data_train, data_num_train, label_train = \
data_utils.grouped_shuffle([data_train, data_num_train, label_train])
num_train = data_train.shape[0]
point_num = data_train.shape[1]
num_val = data_val.shape[0]
num_unseen = data_unseen.shape[0]
batch_num_unseen = int(math.ceil(num_unseen / batch_size))
print('{}-{:d}/{:d} training/validation samples.'.format(
datetime.now(), num_train, num_val))
batch_num = (num_train * num_epochs + batch_size - 1) // batch_size
print('{}-{:d} training batches.'.format(datetime.now(), batch_num))
batch_num_val = int(math.ceil(num_val / batch_size))
print('{}-{:d} testing batches per test.'.format(datetime.now(),
batch_num_val))
######################################################################
# Placeholders
indices = tf.placeholder(tf.int32, shape=(None, None, 2), name="indices")
xforms = tf.placeholder(tf.float32, shape=(None, 3, 3), name="xforms")
rotations = tf.placeholder(tf.float32,
shape=(None, 3, 3),
name="rotations")
jitter_range = tf.placeholder(tf.float32, shape=(1), name="jitter_range")
global_step = tf.Variable(0, trainable=False, name='global_step')
is_training = tf.placeholder(tf.bool, name='is_training')
is_labelled_data = tf.placeholder(tf.bool, name='is_labelled_data')
pts_fts = tf.placeholder(tf.float32,
shape=(None, point_num, setting.data_dim),
name='pts_fts')
labels_seg = tf.placeholder(tf.int64,
shape=(None, point_num),
name='labels_seg')
labels_weights = tf.placeholder(tf.float32,
shape=(None, point_num),
name='labels_weights')
######################################################################
pts_fts_sampled = tf.gather_nd(pts_fts,
indices=indices,
name='pts_fts_sampled')
features_augmented = None
if setting.data_dim > 3:
points_sampled, features_sampled = tf.split(
pts_fts_sampled, [3, setting.data_dim - 3],
axis=-1,
name='split_points_features')
if setting.use_extra_features:
if setting.with_normal_feature:
if setting.data_dim < 6:
print('Only 3D normals are supported!')
exit()
elif setting.data_dim == 6:
features_augmented = pf.augment(features_sampled,
rotations)
else:
normals, rest = tf.split(features_sampled,
[3, setting.data_dim - 6])
normals_augmented = pf.augment(normals, rotations)
features_augmented = tf.concat([normals_augmented, rest],
axis=-1)
else:
features_augmented = features_sampled
else:
points_sampled = pts_fts_sampled
points_augmented = pf.augment(points_sampled, xforms, jitter_range)
labels_sampled = tf.gather_nd(labels_seg,
indices=indices,
name='labels_sampled')
labels_weights_sampled = tf.gather_nd(labels_weights,
indices=indices,
name='labels_weight_sampled')
net = model.Net(points_augmented, features_augmented, is_training, setting)
logits = net.logits
probs = tf.nn.softmax(logits, name='prob')
predictions = tf.argmax(probs, axis=-1, name='predictions')
with tf.variable_scope('xcrf_ker_weights'):
_, point_indices = pf.knn_indices_general(points_augmented,
points_augmented, 1024, True)
xcrf = learningBlock(num_points=sample_num,
num_classes=setting.num_class,
theta_alpha=float(5),
theta_beta=float(2),
theta_gamma=float(1),
num_iterations=5,
name='xcrf',
point_indices=point_indices)
_logits1 = xcrf.call(net.logits, points_augmented, features_augmented,
setting.data_dim)
_logits2 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=2)
_logits3 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=3)
_logits4 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=4)
_logits5 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=8)
_logits6 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=16)
_logits = _logits1 + _logits2 + _logits3 + _logits4 + _logits5 + _logits6
_probs = tf.nn.softmax(_logits, name='probs_crf')
_predictions = tf.argmax(_probs, axis=-1, name='predictions_crf')
logits = tf.cond(
is_training,
lambda: tf.cond(is_labelled_data, lambda: _logits, lambda: logits),
lambda: _logits)
predictions = tf.cond(
is_training, lambda: tf.cond(is_labelled_data, lambda: _predictions,
lambda: predictions),
lambda: _predictions)
labels_sampled = tf.cond(
is_training, lambda: tf.cond(is_labelled_data, lambda: labels_sampled,
lambda: _predictions),
lambda: labels_sampled)
loss_op = tf.losses.sparse_softmax_cross_entropy(
labels=labels_sampled, logits=logits, weights=labels_weights_sampled)
with tf.name_scope('metrics'):
loss_mean_op, loss_mean_update_op = tf.metrics.mean(loss_op)
t_1_acc_op, t_1_acc_update_op = tf.metrics.accuracy(
labels_sampled, predictions, weights=labels_weights_sampled)
t_1_per_class_acc_op, t_1_per_class_acc_update_op = \
tf.metrics.mean_per_class_accuracy(labels_sampled, predictions, setting.num_class,
weights=labels_weights_sampled)
t_1_per_mean_iou_op, t_1_per_mean_iou_op_update_op = \
tf.metrics.mean_iou(labels_sampled, predictions, setting.num_class,
weights=labels_weights_sampled)
reset_metrics_op = tf.variables_initializer([
var for var in tf.local_variables()
if var.name.split('/')[0] == 'metrics'
])
_ = tf.summary.scalar('loss/train',
tensor=loss_mean_op,
collections=['train'])
_ = tf.summary.scalar('t_1_acc/train',
tensor=t_1_acc_op,
collections=['train'])
_ = tf.summary.scalar('t_1_per_class_acc/train',
tensor=t_1_per_class_acc_op,
collections=['train'])
_ = tf.summary.scalar('loss/val', tensor=loss_mean_op, collections=['val'])
_ = tf.summary.scalar('t_1_acc/val',
tensor=t_1_acc_op,
collections=['val'])
_ = tf.summary.scalar('t_1_per_class_acc/val',
tensor=t_1_per_class_acc_op,
collections=['val'])
_ = tf.summary.scalar('t_1_mean_iou/val',
tensor=t_1_per_mean_iou_op,
collections=['val'])
all_variable = tf.global_variables()
lr_exp_op = tf.train.exponential_decay(setting.learning_rate_base,
global_step,
setting.decay_steps,
setting.decay_rate,
staircase=True)
lr_clip_op = tf.maximum(lr_exp_op, setting.learning_rate_min)
_ = tf.summary.scalar('learning_rate',
tensor=lr_clip_op,
collections=['train'])
reg_loss = setting.weight_decay * tf.losses.get_regularization_loss()
if setting.optimizer == 'adam':
optimizer = tf.train.AdamOptimizer(learning_rate=lr_clip_op,
epsilon=setting.epsilon)
elif setting.optimizer == 'momentum':
optimizer = tf.train.MomentumOptimizer(learning_rate=lr_clip_op,
momentum=setting.momentum,
use_nesterov=True)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
xcrf_ker_weights = [
var for var in tf.global_variables() if 'xcrf_ker_weights' in var.name
]
no_xcrf_ker_weights = [
var for var in tf.global_variables()
if 'xcrf_ker_weights' not in var.name
]
#print(restore_values)
#train_op = optimizer.minimize( loss_op+reg_loss, global_step=global_step)
with tf.control_dependencies(update_ops):
train_op_xcrf = optimizer.minimize(loss_op + reg_loss,
var_list=no_xcrf_ker_weights,
global_step=global_step)
train_op_all = optimizer.minimize(loss_op + reg_loss,
var_list=all_variable,
global_step=global_step)
train_op = tf.cond(is_labelled_data, lambda: train_op_all,
lambda: train_op_xcrf)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
saver = tf.train.Saver(var_list=no_xcrf_ker_weights, max_to_keep=5)
saver_best = tf.train.Saver(max_to_keep=5)
# backup all code
code_folder = os.path.abspath(os.path.dirname(__file__))
shutil.copytree(code_folder,
os.path.join(root_folder, os.path.basename(code_folder)))
folder_ckpt = os.path.join(root_folder, 'ckpts')
if not os.path.exists(folder_ckpt):
os.makedirs(folder_ckpt)
folder_ckpt_best = os.path.join(root_folder, 'ckpt-best')
if not os.path.exists(folder_ckpt_best):
os.makedirs(folder_ckpt_best)
folder_summary = os.path.join(root_folder, 'summary')
if not os.path.exists(folder_summary):
os.makedirs(folder_summary)
parameter_num = np.sum(
[np.prod(v.shape.as_list()) for v in tf.trainable_variables()])
print('{}-Parameter number: {:d}.'.format(datetime.now(),
int(parameter_num)))
_highest_val = 0.0
max_val = 10
with tf.Session() as sess:
summaries_op = tf.summary.merge_all('train')
summaries_val_op = tf.summary.merge_all('val')
summary_values_op = tf.summary.merge_all('summary_values')
summary_writer = tf.summary.FileWriter(folder_summary, sess.graph)
sess.run(init_op)
# Load the model
if args.load_ckpt is not None:
saver.restore(sess, args.load_ckpt)
print('{}-Checkpoint loaded from {}!'.format(
datetime.now(), args.load_ckpt))
batch_num = 50000
for batch_idx_train in range(batch_num):
if (batch_idx_train % step_val == 0 and (batch_idx_train != 0 or args.load_ckpt is not None)) \
or batch_idx_train == batch_num - 1:
######################################################################
# Validation
filename_ckpt = os.path.join(folder_ckpt, 'iter')
saver.save(sess, filename_ckpt, global_step=global_step)
print('{}-Checkpoint saved to {}!'.format(
datetime.now(), filename_ckpt))
sess.run(reset_metrics_op)
summary_hist = None
_idxVal = np.arange(num_val)
np.random.shuffle(_idxVal)
_pred = []
_label = []
for batch_val_idx in tqdm(range(batch_num_val)):
start_idx = batch_size * batch_val_idx
end_idx = min(start_idx + batch_size, num_val)
batch_size_val = end_idx - start_idx
points_batch = data_val[_idxVal[start_idx:end_idx], ...]
points_num_batch = data_num_val[_idxVal[start_idx:end_idx],
...]
labels_batch = label_val[_idxVal[start_idx:end_idx], ...]
weights_batch = np.array(label_weights_val)[labels_batch]
xforms_np, rotations_np = pf.get_xforms(
batch_size_val,
rotation_range=rotation_range_val,
scaling_range=scaling_range_val,
order=setting.rotation_order)
_labels_sampled, _predictions, _, _, _, _ = sess.run(
[
labels_sampled, predictions, loss_mean_update_op,
t_1_acc_update_op, t_1_per_class_acc_update_op,
t_1_per_mean_iou_op_update_op
],
feed_dict={
pts_fts:
points_batch,
indices:
pf.get_indices(batch_size_val, sample_num,
points_num_batch),
xforms:
xforms_np,
rotations:
rotations_np,
jitter_range:
np.array([jitter_val]),
labels_seg:
labels_batch,
is_labelled_data:
True,
labels_weights:
weights_batch,
is_training:
False,
})
_pred.append(_predictions.flatten())
_label.append(_labels_sampled.flatten())
loss_val, t_1_acc_val, t_1_per_class_acc_val, t1__mean_iou, summaries_val = sess.run(
[
loss_mean_op, t_1_acc_op, t_1_per_class_acc_op,
t_1_per_mean_iou_op, summaries_val_op
])
img_d_summary = pf.plot_confusion_matrix(
_label,
_pred, ["Environment", "Pedestrian", "Car", "Cyclist"],
tensor_name='confusion_matrix')
max_val = max_val - 1
if (t_1_per_class_acc_val > _highest_val):
max_val = 10
_highest_val = t_1_per_class_acc_val
filename_ckpt = os.path.join(folder_ckpt_best,
str(_highest_val) + "-iter-")
saver_best.save(sess,
filename_ckpt,
global_step=global_step)
if (max_val < 0):
sys.exit(0)
summary_writer.add_summary(summaries_val, batch_idx_train)
summary_writer.add_summary(img_d_summary, batch_idx_train)
#summary_writer.add_summary(summary_hist, batch_idx_train)
print(
'{}-[Val ]-Average: Loss: {:.4f} T-1 Acc: {:.4f} Diff-Best: {:.4f} T-1 mAcc: {:.4f} T-1 mIoU: {:.4f}'
.format(datetime.now(), loss_val, t_1_acc_val,
_highest_val - t_1_per_class_acc_val,
t_1_per_class_acc_val, t1__mean_iou))
sys.stdout.flush()
######################################################################
# Unseen Data
sess.run(reset_metrics_op)
summary_hist = None
_idxunseen = np.arange(num_unseen)
np.random.shuffle(_idxunseen)
_pred = []
_label = []
unseenIndices = np.arange(batch_num_unseen)
np.random.shuffle(unseenIndices)
for batch_val_idx in tqdm(unseenIndices[:500]):
start_idx = batch_size * batch_val_idx
end_idx = min(start_idx + batch_size, num_unseen)
batch_size_train = end_idx - start_idx
points_batch = data_unseen[_idxunseen[start_idx:end_idx],
...]
points_num_batch = data_num_unseen[
_idxunseen[start_idx:end_idx], ...]
labels_batch = np.zeros(points_batch.shape[0:2],
dtype=np.int32)
weights_batch = np.array(label_weights_list)[labels_batch]
xforms_np, rotations_np = pf.get_xforms(
batch_size_train,
rotation_range=rotation_range,
scaling_range=scaling_range,
order=setting.rotation_order)
offset = int(
random.gauss(0,
sample_num * setting.sample_num_variance))
offset = max(offset, -sample_num * setting.sample_num_clip)
offset = min(offset, sample_num * setting.sample_num_clip)
sample_num_train = sample_num + offset
sess.run(
[
train_op, loss_mean_update_op, t_1_acc_update_op,
t_1_per_class_acc_update_op,
t_1_per_mean_iou_op_update_op
],
feed_dict={
pts_fts:
points_batch,
is_labelled_data:
False,
indices:
pf.get_indices(batch_size_train, sample_num_train,
points_num_batch),
xforms:
xforms_np,
rotations:
rotations_np,
jitter_range:
np.array([jitter]),
labels_seg:
labels_batch,
labels_weights:
weights_batch,
is_training:
True,
})
if batch_val_idx % 100 == 0:
loss, t_1_acc, t_1_per_class_acc, t_1__mean_iou = sess.run(
[
loss_mean_op, t_1_acc_op, t_1_per_class_acc_op,
t_1_per_mean_iou_op
],
feed_dict={
pts_fts:
points_batch,
indices:
pf.get_indices(batch_size_train,
sample_num_train,
points_num_batch),
xforms:
xforms_np,
is_labelled_data:
False,
rotations:
rotations_np,
jitter_range:
np.array([jitter]),
labels_seg:
labels_batch,
labels_weights:
weights_batch,
is_training:
True,
})
print(
'{}-[Train]-Unseen: {:06d} Loss: {:.4f} T-1 Acc: {:.4f} T-1 mAcc: {:.4f} T-1 mIoU: {:.4f}'
.format(datetime.now(), batch_val_idx, loss,
t_1_acc, t_1_per_class_acc, t_1__mean_iou))
sys.stdout.flush()
######################################################################
# Training
start_idx = (batch_size * batch_idx_train) % num_train
end_idx = min(start_idx + batch_size, num_train)
batch_size_train = end_idx - start_idx
points_batch = data_train[start_idx:end_idx, ...]
points_num_batch = data_num_train[start_idx:end_idx, ...]
labels_batch = label_train[start_idx:end_idx, ...]
weights_batch = np.array(label_weights_list)[labels_batch]
if start_idx + batch_size_train == num_train:
if is_list_of_h5_list:
filelist_train_prev = seg_list[(seg_list_idx - 1) %
len(seg_list)]
filelist_train = seg_list[seg_list_idx % len(seg_list)]
if filelist_train != filelist_train_prev:
data_train, _, data_num_train, label_train, _ = data_utils.load_seg(
filelist_train)
num_train = data_train.shape[0]
seg_list_idx = seg_list_idx + 1
data_train, data_num_train, label_train = \
data_utils.grouped_shuffle([data_train, data_num_train, label_train])
offset = int(
random.gauss(0, sample_num * setting.sample_num_variance))
offset = max(offset, -sample_num * setting.sample_num_clip)
offset = min(offset, sample_num * setting.sample_num_clip)
sample_num_train = sample_num + offset
xforms_np, rotations_np = pf.get_xforms(
batch_size_train,
rotation_range=rotation_range,
scaling_range=scaling_range,
order=setting.rotation_order)
sess.run(reset_metrics_op)
sess.run(
[
train_op, loss_mean_update_op, t_1_acc_update_op,
t_1_per_class_acc_update_op, t_1_per_mean_iou_op_update_op
],
feed_dict={
pts_fts:
points_batch,
is_labelled_data:
True,
indices:
pf.get_indices(batch_size_train, sample_num_train,
points_num_batch),
xforms:
xforms_np,
rotations:
rotations_np,
jitter_range:
np.array([jitter]),
labels_seg:
labels_batch,
labels_weights:
weights_batch,
is_training:
True,
})
if batch_idx_train % 100 == 0:
loss, t_1_acc, t_1_per_class_acc, t_1__mean_iou, summaries = sess.run(
[
loss_mean_op, t_1_acc_op, t_1_per_class_acc_op,
t_1_per_mean_iou_op, summaries_op
],
feed_dict={
pts_fts:
points_batch,
indices:
pf.get_indices(batch_size_train, sample_num_train,
points_num_batch),
xforms:
xforms_np,
is_labelled_data:
True,
rotations:
rotations_np,
jitter_range:
np.array([jitter]),
labels_seg:
labels_batch,
labels_weights:
weights_batch,
is_training:
True,
})
summary_writer.add_summary(summaries, batch_idx_train)
print(
'{}-[Train]-Iter: {:06d} Loss: {:.4f} T-1 Acc: {:.4f} T-1 mAcc: {:.4f} T-1 mIoU: {:.4f}'
.format(datetime.now(), batch_idx_train, loss, t_1_acc,
t_1_per_class_acc, t_1__mean_iou))
sys.stdout.flush()
######################################################################
print('{}-Done!'.format(datetime.now()))
def ficonv(pts,
fts,
qrs,
tag,
N,
K1,
mm,
sigma,
scale,
K,
D,
P,
C,
C_pts_fts,
kernel_num,
is_training,
with_kernel_registering,
with_kernel_shape_comparison,
with_point_transformation,
with_feature_transformation,
with_learning_feature_transformation,
kenel_initialization_method,
depth_multiplier,
sorting_method=None,
with_global=False):
Dis, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + 'nn_pts_local') # (N, P, K, 3)
if with_point_transformation or with_feature_transformation:
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK,
K,
tag + 'X_2',
is_training, (1, K),
activation=None)
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
if with_point_transformation:
if with_learning_feature_transformation:
nn_pts_local = tf.matmul(X_2_KK, nn_pts_local)
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
else:
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
nn_pts_local = tf.matmul(X_2_KK, nn_pts_local)
else:
if with_learning_feature_transformation:
nn_pts_local_ = tf.matmul(X_2_KK,
nn_pts_local,
name=tag + 'nn_pts_local_')
# Prepare features to be transformed
nn_fts_from_pts_0 = pf.dense(nn_pts_local_, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
else:
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts,
tag + 'nn_fts_from_pts_0',
is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts,
tag + 'nn_fts_from_pts', is_training)
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts,
indices,
name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev],
axis=-1,
name=tag + 'nn_fts_input')
P1 = tf.shape(nn_pts_local)[1]
dim1 = 3
if with_kernel_registering:
######################## preparing #########################
if with_feature_transformation:
nn_fts_input = tf.matmul(X_2_KK, nn_fts_input)
r_data = tf.reduce_sum(nn_pts_local * nn_pts_local,
axis=3,
keep_dims=True,
name=tag + 'kernel_pow')
######################## kernel-registering #########################
shape_id = 0
if kenel_initialization_method == 'random':
kernel_shape = tf.Variable(tf.random_uniform([K1, dim1],
minval=-0.5,
maxval=0.5,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
else:
kernel_shape = tf.Variable(tf.random_normal([K1, dim1],
mean=0.0,
stddev=1.0,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
kernel_shape_dis = tf.sqrt(tf.reduce_sum(kernel_shape * kernel_shape,
axis=1),
name=tag + 'kernel_shape_dis' +
str(shape_id))
kernel_shape_normal = scale * tf.div(
kernel_shape,
tf.reduce_max(kernel_shape_dis),
name=tag + 'kernel_shape_normal' + str(shape_id))
r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal,
axis=1,
keep_dims=True,
name=tag + 'kernel_pow' + str(shape_id))
reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1],
name=tag + 'reshape_kernel' + str(shape_id))
m = tf.reshape(tf.matmul(reshape_data,
tf.transpose(kernel_shape_normal)),
[N, P1, K, K1],
name=tag + 'mm' + str(shape_id))
dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel),
perm=[0, 1, 3, 2],
name=tag + 'dis_matrix' + str(shape_id))
coef_matrix = tf.exp(tf.div(-dis_matrix, sigma),
name=tag + 'coef_matrix' + str(shape_id))
#coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id))
if with_kernel_shape_comparison:
coef_global = tf.reduce_sum(
coef_matrix, axis=[2, 3], keep_dims=True) / K
coef_normal = coef_global * tf.div(
coef_matrix,
tf.reduce_sum(coef_matrix, axis=3, keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
else:
coef_normal = tf.div(coef_matrix,
tf.reduce_sum(coef_matrix,
axis=3,
keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
fts_X = tf.matmul(coef_normal,
nn_fts_input,
name=tag + 'fts_X' + str(shape_id))
###################################################################
fts_conv = pf.separable_conv2d(fts_X,
math.ceil(mm * C / kernel_num),
tag + 'fts_conv' + str(shape_id),
is_training, (1, K1),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv,
axis=2,
name=tag + 'fts_conv_3d' + str(shape_id))
for shape_id in range(kernel_num - 1):
shape_id = shape_id + 1
if kenel_initialization_method == 'random':
kernel_shape = tf.Variable(tf.random_uniform([K1, dim1],
minval=-0.5,
maxval=0.5,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
else:
kernel_shape = tf.Variable(tf.random_normal([K1, dim1],
mean=0.0,
stddev=1.0,
dtype=tf.float32),
name=tag + 'kernel_shape' +
str(shape_id))
kernel_shape_dis = tf.sqrt(
tf.reduce_sum(kernel_shape * kernel_shape, axis=1),
name=tag + 'kernel_shape_dis' + str(shape_id))
kernel_shape_normal = scale * tf.div(
kernel_shape,
tf.reduce_max(kernel_shape_dis),
name=tag + 'kernel_shape_normal' + str(shape_id))
r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal,
axis=1,
keep_dims=True,
name=tag + 'kernel_pow' + str(shape_id))
reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1],
name=tag + 'reshape_kernel' +
str(shape_id))
m = tf.reshape(tf.matmul(reshape_data,
tf.transpose(kernel_shape_normal)),
[N, P1, K, K1],
name=tag + 'mm' + str(shape_id))
dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel),
perm=[0, 1, 3, 2],
name=tag + 'dis_matrix' + str(shape_id))
coef_matrix = tf.exp(tf.div(-dis_matrix, sigma),
name=tag + 'coef_matrix' + str(shape_id))
#coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id))
if with_kernel_shape_comparison:
coef_global = tf.reduce_sum(
coef_matrix, axis=[2, 3], keep_dims=True) / K
coef_normal = coef_global * tf.div(
coef_matrix,
tf.reduce_sum(coef_matrix, axis=3, keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
else:
coef_normal = tf.div(coef_matrix,
tf.reduce_sum(coef_matrix,
axis=3,
keep_dims=True),
name=tag + 'coef_normal' + str(shape_id))
fts_X = tf.matmul(coef_normal,
nn_fts_input,
name=tag + 'fts_X' + str(shape_id))
###################################################################
fts_conv = pf.separable_conv2d(fts_X,
math.ceil(mm * C / kernel_num),
tag + 'fts_conv' + str(shape_id),
is_training, (1, K1),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.concat(
[fts_conv_3d, tf.squeeze(fts_conv, axis=2)],
axis=-1,
name=tag + 'fts_conv_3d' + str(shape_id))
else:
fts_X = nn_fts_input
fts_conv = pf.separable_conv2d(fts_X,
C,
tag + 'fts_conv',
is_training, (1, K),
depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global',
is_training)
return tf.concat([fts_global, fts_conv_3d],
axis=-1,
name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def point_conv_transpose_1(pts_in,
fts_in,
pts_out,
indices,
K,
D,
C,
is_training,
activation=tf.nn.relu,
bn=True,
sorting_method='cxyz',
tag='point_conv'):
'''
pts_in: xyz points, Tensor, (batch_size, npoint, 3)
fts_in: features, Tensor, (batch_size, npoint, channel_fts)
pts_out: xyz points, Tensor, (batch_size, n_pts_out, 3), NOTE: pts_out is denser than pts.
K: number of neighbors for one query, scalar, int
D: dilation rate
C: output feature channel number
in the comments:
N - batch size
P - number of pts_in
K - number of neighbors
'''
batch_size = pts_out.shape[0]
n_pts_out = pts_out.shape[1]
n_pts_in = pts_in.shape[1]
if indices == None:
_, indices_dilated = pf.knn_indices_general(pts_in, pts_out, K * D,
True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts_out, indices,
sorting_method) # (N, P, K, 2)
# move the gathered pts_out to local frame
nn_pts = tf.gather_nd(pts_out, indices,
name=tag + '_nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(pts_in, axis=2,
name=tag + '_nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + '_nn_pts_local') # (N, P, K, 3)
if fts_in is None:
#nn_fts_input = pts_in # (N, P, 3)
print('Error: fts_in is None.')
else:
#nn_fts_input = tf.concat([pts_in, fts_in], axis=-1, name=tag + 'nn_fts_input') # (N, P, 3+C_fts_in)
nn_fts_input = fts_in # (N, P, C_fts_in)
nn_fts_input = tf.expand_dims(nn_fts_input, axis=2) # (N, P, 1, C_fts_in)
# TODO: maybe try other conv1d? fc? there
fts_conv = pf.conv2d_transpose(
nn_fts_input,
C,
tag + '_fts_deconv',
is_training,
kernel_size=(1, 1),
strides=(1, K),
with_bn=bn,
activation=activation) # (N, P, 1, C_fts_in) -> (N, P, K, C)
fts_conv_pts_out_concat = tf.concat([fts_conv, nn_pts_local],
axis=-1) # (N, P, K, C+3)
fts_pts_conv = pf.conv2d(
fts_conv_pts_out_concat,
C,
tag + '_fts_pts_conv1D',
is_training,
kernel_size=(1, 1),
strides=(1, 1),
with_bn=bn,
activation=activation) # (N, P, K, C+3) -> (N, P, K, C)
######## collect fts for points in the output #######################
##########################################################################
npoint_indices = tf.reshape(tf.range(n_pts_in), [-1, 1]) # (P, 1)
npoint_indices = tf.tile(npoint_indices, (1, K)) # (P, K)
npoint_indices = tf.expand_dims(npoint_indices, axis=0) # (1, P, K)
npoint_indices = tf.tile(npoint_indices, (batch_size, 1, 1)) # (N, P, K)
npoint_indices = tf.expand_dims(npoint_indices, axis=3) # (N, P, K, 1)
NPK_indices = tf.concat(
[
tf.expand_dims(indices[:, :, :, 0], axis=-1), npoint_indices,
tf.expand_dims(indices[:, :, :, 1], axis=-1)
],
axis=3
) # (N, P, K, 3) last dim is 3 for (batch_idx, point_idx, neighbor_idx)
NPK_indices = tf.expand_dims(NPK_indices, axis=3) # (N, P, K, 1, 3)
NPK_indices = tf.tile(NPK_indices, (1, 1, 1, C, 1)) # (N, P, K, C, 3)
channel_indices = tf.reshape(tf.range(C),
[1, 1, 1, -1, 1]) # (1, 1, 1, C, 1)
channel_indices = tf.tile(
channel_indices, (batch_size, n_pts_in, K, 1, 1)) # (N, P, K, C, 1)
final_indices = tf.concat(
[NPK_indices, channel_indices], axis=4
) # (N, P, K, C, 4) last dim: (batch_idx, rep_point_idx, neighbor_idx_in_pts_out, channel_idx)
######## reshape for constructing sparse tensor ################
final_indices = tf.reshape(final_indices, [-1, 4])
final_value = tf.reshape(fts_pts_conv, [-1])
# using sparse tensor and sparse ops
# TODO: in tf 1.4, the sparse_reduce_sum cannot infer the shape even if the input shape is known.
# so, try to use latest tf, a version of after Oct. 2018.
#fts_sparse = tf.SparseTensor(indices=tf.cast(final_indices, tf.int64), values=final_value, dense_shape=[batch_size, n_pts_in, n_pts_out, C]) # (N, P, N_PTS_OUT, C) in sparse representation
#fts_out = tf.sparse_reduce_sum(fts_sparse, axis=1) # (N, N_PTS_OUT, C)
#fts_dense = tf.sparse_tensor_to_dense(fts_sparse, default_value=0, validate_indices=False) # (N, P, N_PTS_OUT, C)
# using dense tensor, memory-consuming
fts_dense = tf.scatter_nd(
final_indices, final_value,
[batch_size, n_pts_in, n_pts_out, C]) # (N, P, N_PTS_OUT, C)
fts_out = tf.reduce_sum(fts_dense, axis=1) # (N, N_PTS_OUT, C)
return fts_out
def main(FLAG):
data_filename = FLAG.filename
retrieve_whole_files = FLAG.retrieve_whole_files
CHECKPOINT_LOCATION = "/home/hasan/data/hdd8TB/KITTI/3D-object-detection/pointcnn-model/AXCRF-random-XYZ-grid-0.25/pointcnn_seg_kitti3d_x8_2048_fps_2019-04-11-14-43-24_31566/ckpt-best/0.9320669-iter--237975"
max_point_num = 8192
# Root directory of the project
# data_filename = FLAGS.image_filename
ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
print(ROOT_DIR)
# Directory to save logs and trained model
MODEL_DIR = os.path.join(ROOT_DIR, "logs")
# Directory to get binary and image data
PARENT_DIR = os.path.abspath(os.path.join(ROOT_DIR, os.pardir))
DATA_DIR = os.path.join(PARENT_DIR, "test_dataset")
## Feed data to pointCNN
args_load_ckpt = CHECKPOINT_LOCATION
args_model = "pointcnn_seg"
args_setting = "kitti3d_x8_2048_fps"
args_repeat_num = 1
model = importlib.import_module(args_model)
setting_path = os.path.join(os.path.dirname(__file__), args_model)
sys.path.append(setting_path)
setting = importlib.import_module(args_setting)
sample_num = setting.sample_num
batch_size = args_repeat_num * int(math.ceil(max_point_num / sample_num))
######################################################################
# Placeholders
indices = tf.placeholder(tf.int32,
shape=(batch_size, None, 2),
name="indices")
is_training = tf.placeholder(tf.bool, name='is_training')
pts_fts = tf.placeholder(tf.float32,
shape=(batch_size, max_point_num,
setting.data_dim),
name='points')
######################################################################
######################################################################
pts_fts_sampled = tf.gather_nd(pts_fts,
indices=indices,
name='pts_fts_sampled')
if setting.data_dim > 3:
points_sampled, features_sampled = tf.split(
pts_fts_sampled, [3, setting.data_dim - 3],
axis=-1,
name='split_points_features')
if not setting.use_extra_features:
features_sampled = None
else:
points_sampled = pts_fts_sampled
features_sampled = None
net = model.Net(points_sampled, features_sampled, is_training, setting)
points_augmented = points_sampled
features_augmented = features_sampled
with tf.variable_scope('xcrf_ker_weights'):
_, point_indices = pf.knn_indices_general(points_augmented,
points_augmented, 1024, True)
xcrf = learningBlock(num_points=sample_num,
num_classes=setting.num_class,
theta_alpha=float(5),
theta_beta=float(2),
theta_gamma=float(1),
num_iterations=10,
name='xcrf',
point_indices=point_indices)
_logits1 = xcrf.call(net.logits, points_augmented, features_augmented,
setting.data_dim)
_logits2 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=2)
_logits3 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=3)
_logits4 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=4)
_logits5 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=8)
_logits6 = xcrf.call(net.logits,
points_augmented,
features_augmented,
setting.data_dim,
D=16)
_logits = _logits1 + _logits2 + _logits3 + _logits4 + _logits5 + _logits6
seg_probs_op = tf.nn.softmax(_logits, name='seg_probs')
# for restore model
saver = tf.train.Saver()
parameter_num = np.sum(
[np.prod(v.shape.as_list()) for v in tf.trainable_variables()])
print('{}-Parameter number: {:d}.'.format(datetime.now(), parameter_num))
with tf.Session() as sess:
# Load the model
saver.restore(sess, args_load_ckpt)
print('{}-Checkpoint loaded from {}!'.format(datetime.now(),
args_load_ckpt))
indices_batch_indices = np.tile(
np.reshape(np.arange(batch_size), (batch_size, 1, 1)),
(1, sample_num, 1))
drivename, fname = data_filename.split("/")
all_files = [fname]
if (retrieve_whole_files):
source_path = os.path.join(DATA_DIR, drivename, "bin_data")
glob_files = (glob.glob(source_path + "/*.bin"))
all_files = [fname] + [
_i.replace(source_path + "/", "")[0:-4] for _i in glob_files
]
for fname in all_files:
output_location = os.path.join(
ROOT_DIR, "output/" + drivename + "_" + fname + "axcrf.bin")
if os.path.isfile(output_location):
continue
label_length, data, data_num, indices_split_to_full, item_num, all_label_pred, indices_for_prediction = data_preprocessing(
drivename, fname, max_point_num)
merged_label_zero = np.zeros((label_length), dtype=int)
merged_confidence_zero = np.zeros((label_length), dtype=float)
data = data[0:item_num, ...].astype(np.float32)
data_num = data_num[0:item_num, ...]
indices_split_to_full = indices_split_to_full[0:item_num]
batch_num = data.shape[0]
labels_pred = np.full((batch_num, max_point_num),
-1,
dtype=np.int32)
confidences_pred = np.zeros((batch_num, max_point_num),
dtype=np.float32)
for batch_idx in range(batch_num):
points_batch = data[[batch_idx] * batch_size, ...]
point_num = data_num[batch_idx]
tile_num = int(math.ceil(
(sample_num * batch_size) / point_num))
indices_shuffle = np.tile(np.arange(point_num),
tile_num)[0:sample_num * batch_size]
np.random.shuffle(indices_shuffle)
indices_batch_shuffle = np.reshape(indices_shuffle,
(batch_size, sample_num, 1))
indices_batch = np.concatenate(
(indices_batch_indices, indices_batch_shuffle), axis=2)
seg_probs = sess.run(
[seg_probs_op],
feed_dict={
pts_fts: points_batch,
indices: indices_batch.astype(np.int32),
is_training: False,
})
probs_2d = np.reshape(seg_probs, (sample_num * batch_size, -1))
predictions = [(-1, 0.0)] * point_num
for idx in range(sample_num * batch_size):
point_idx = indices_shuffle[idx]
probs = probs_2d[idx, :]
confidence = np.amax(probs)
label = np.argmax(probs)
if confidence > predictions[point_idx][1]:
predictions[point_idx] = [label, confidence]
labels_pred[batch_idx, 0:point_num] = np.array(
[label for label, _ in predictions])
confidences_pred[batch_idx, 0:point_num] = np.array(
[confidence for _, confidence in predictions])
for idx in range(batch_num): #Get highest confidence
pred = labels_pred[idx].astype(np.int64)
_indices = indices_split_to_full[idx].astype(np.int64)
confidence = confidences_pred[idx].astype(np.float32)
num = data_num[idx].astype(np.int64)
for i in range(pred.shape[0]):
if confidence[i] > 0.8 and confidence[
i] > merged_confidence_zero[_indices[i]]:
merged_confidence_zero[_indices[i]] = confidence[i]
merged_label_zero[_indices[i]] = pred[i]
print('{}-Done!'.format(datetime.now()))
all_label_pred[indices_for_prediction] = merged_label_zero
bounded_indices = (all_label_pred == 2).nonzero()[0]
bounded_indices.tofile(output_location)
return
#Remove Ground points
"""
def __init__(self, points, features, num_class, is_training, setting, task):
xconv_params = setting.xconv_params
fc_params = setting.fc_params
with_X_transformation = setting.with_X_transformation
sorting_method = setting.sorting_method
N = tf.shape(points)[0]
fts_T,fts_T1 = pointnet1(points, is_training, N, PP=1024)
if setting.with_fps:
from sampling import tf_sampling
self.layer_pts = [points]
qrs_indices = pf.knn_indices_general(points, points, 1, False)
self.layer_qrs_indices = [qrs_indices]
if features is None:
self.layer_fts = [features]
else:
C_fts = xconv_params[0][-1] // 2
features_hd = pf.dense(features, C_fts, 'features_hd', is_training)
self.layer_fts = [features_hd]
for layer_idx, layer_param in enumerate(xconv_params):
tag = 'xconv_' + str(layer_idx + 1) + '_'
K, D, P, C = layer_param
# get k-nearest points
pts = self.layer_pts[-1]
fts = self.layer_fts[-1]
if P == -1:
qrs = points
_, qrs_indices = pf.knn_indices_general(qrs, points, 1, False) #### zj 2018/04/10
else:
if setting.with_fps:
qrs = tf_sampling.gather_point(pts, qrs_indices) # (N,P,3)
else:
qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3)
_, qrs_indices = pf.knn_indices_general(qrs, points, 1, False) #### zj 2018/04/10
print(qrs_indices) #### zj 2018/04/10
self.layer_qrs_indices.append(qrs_indices)
self.layer_pts.append(qrs)
if layer_idx == 0:
C_pts_fts = C // 2 if fts is None else C // 4
depth_multiplier = 4
else:
C_prev = xconv_params[layer_idx - 1][-1]
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation,
depth_multiplier, sorting_method)
self.layer_fts.append(fts_xconv)
if task == 'segmentation':
for layer_idx, layer_param in enumerate(setting.xdconv_params):
tag = 'xdconv_' + str(layer_idx + 1) + '_'
K, D, pts_layer_idx, qrs_layer_idx = layer_param
pts = self.layer_pts[pts_layer_idx + 1]
fts = self.layer_fts[pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1]
qrs = self.layer_pts[qrs_layer_idx + 1]
fts_qrs = self.layer_fts[qrs_layer_idx + 1]
_, _, P, C = xconv_params[qrs_layer_idx]
_, _, _, C_prev = xconv_params[pts_layer_idx]
C_pts_fts = C_prev // 4
depth_multiplier = 1
fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation,
depth_multiplier, sorting_method)
fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat')
fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training)
self.layer_pts.append(qrs)
self.layer_fts.append(fts_fuse)
#################################### zj,2018/04/10####################################
self.fc_layers1 = self.layer_fts[-1]
self.fc_layers_indices = self.layer_qrs_indices[-1]
#last_ftsT = tf.gather_nd(fts_T, self.fc_layers_indices , name='last_ftsT')
#last_ftsT = tf.squeeze(last_ftsT, axis=2, name='last_ftsT1')
nn = tf.shape(self.fc_layers1)[1]
last_ftsT = tf.tile(fts_T, (1,nn,1),name = 'last_ftsT')
self.fc_layers1 = tf.concat([self.fc_layers1, last_ftsT] ,axis = -1, name='last_ftsT_concat')
self.fc_layers = [self.fc_layers1]
print(self.fc_layers)
#######################################################################################
for layer_idx, layer_param in enumerate(fc_params):
channel_num, drop_rate = layer_param
fc = pf.dense(self.fc_layers[-1], channel_num, 'fc{:d}'.format(layer_idx), is_training) #[N,128,channel_num]???
fc_drop = tf.layers.dropout(fc, drop_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx))
self.fc_layers.append(fc_drop)
logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None) #[N,128,40?]???
if task == 'classification':
logits_mean = tf.reduce_mean(logits, axis=1, keep_dims=True, name='logits_mean')
self.logits = tf.cond(is_training, lambda: logits, lambda: logits_mean)
elif task == 'segmentation':
self.logits = logits
else:
print('Unknown task!')
exit()
self.probs = tf.nn.softmax(self.logits, name='probs')
def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier,
sorting_method=None, with_global=False):
'''
pts: batch points input
fts: batch feature inputs
qrs: points outputs
tag: tags
N: batch_size
K: k neighbors, like kernel size in normal conv2d
D: dilation rate
P: sample points size in one sample
C: input channel num
C_pts_fts: output channel num
'''
# get k*d near points'indices from pts according to qrs
_, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True)
# select k points' indices
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts, indices, sorting_method)
# get the k points' data according to the indices
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
# expand qrs from (N, P, 3) to (N, P, 1, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3)
# remove qrs from pts
nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3)
# Prepare features to be transformed
# step 2, MLP
nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training)
nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training)
# step 3, concat origin features with the new features
if fts is None:
nn_fts_input = nn_fts_from_pts
else:
nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev')
nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input')
# step 4, according to the appendix, the second MLP is formes with two depthwise_conv layers
if with_X_transformation:
######################## X-transformation #########################
X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K))
X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK')
X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K))
X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK')
X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None)
X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK')
# step 5
fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X')
###################################################################
else:
fts_X = nn_fts_input
# step 6
fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier)
fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d')
if with_global:
fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training, with_bn=False)
fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training, with_bn=False)
#fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training)
#fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training)
return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global')
else:
return fts_conv_3d
def point_conv_transpose_2(pts_in,
fts_in,
pts_out,
fts_out_skipped,
indices,
K,
D,
C,
is_training,
activation=tf.nn.relu,
bn=True,
sorting_method='cxyz',
tag='point_conv'):
'''
pts_in: xyz points, Tensor, (batch_size, npoint, 3)
fts_in: features, Tensor, (batch_size, npoint, channel_fts)
pts_out: xyz points, Tensor, (batch_size, n_pts_out, 3), NOTE: pts_out is denser than pts.
K: number of neighbors for one query, scalar, int
D: dilation rate
C: output feature channel number
in the comments:
N - batch size
P - number of pts_in
K - number of neighbors
'''
batch_size = pts_out.shape[0]
n_pts_out = pts_out.shape[1]
n_pts_in = pts_in.shape[1]
if indices is None:
_, indices_dilated = pf.knn_indices_general(pts_in, pts_out, K * D,
True)
indices = indices_dilated[:, :, ::D, :]
if sorting_method is not None:
indices = pf.sort_points(pts_out, indices,
sorting_method) # (N, P, K, 2)
# move the gathered pts_out to local frame
nn_pts_out = tf.gather_nd(pts_out, indices,
name=tag + 'nn_pts_out') # (N, P, K, 3)
nn_pts_out_center = tf.expand_dims(pts_in,
axis=2,
name=tag +
'nn_pts_out_center') # (N, P, 1, 3)
nn_pts_out_local = tf.subtract(nn_pts_out,
nn_pts_out_center,
name=tag +
'nn_pts_out_local') # (N, P, K, 3)
if fts_in is None:
nn_fts_input = pts_in # (N, P, 3)
#print('Error: fts_in is None.')
else:
#nn_fts_input = tf.concat([pts_in, fts_in], axis=-1, name=tag + 'nn_fts_input') # (N, P, 3+C_fts_in)
nn_fts_input = fts_in # (N, P, C_fts_in)
nn_fts_input = tf.expand_dims(nn_fts_input, axis=2) # (N, P, 1, C_fts_in)
# TODO: maybe try other conv1d? fc? there
fts_conv = pf.conv2d_transpose(
nn_fts_input,
C,
tag + 'fts_deconv',
is_training,
kernel_size=(1, 1),
strides=(1, K),
with_bn=bn,
activation=activation) # (N, P, 1, C_fts_in) -> (N, P, K, C)
if fts_out_skipped is not None:
nn_fts_out_skipped = tf.gather_nd(fts_out_skipped,
indices,
name=tag + 'nn_fts_out_from_skip')
fts_all_concat = tf.concat(
[fts_conv, nn_fts_out_skipped, nn_pts_out_local],
axis=-1) # (N, P, K, C+3+channel_skipped)
else:
fts_all_concat = tf.concat([fts_conv, nn_pts_out_local],
axis=-1) # (N, P, K, C+3)
fts_pts_conv = pf.conv2d(
fts_all_concat,
C,
tag + 'fts_pts_conv1D',
is_training,
kernel_size=(1, 1),
strides=(1, 1),
with_bn=bn,
activation=activation) # (N, P, K, C+3_ch_skip?) -> (N, P, K, C)
# summing up feature for each point
fts_dense = tf.scatter_nd(tf.reshape(indices, (-1, 2)),
tf.reshape(fts_pts_conv, (-1, C)),
(batch_size, n_pts_out, C)) # (N, N_PTS_OUT, c)
return fts_dense