def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method=None, with_global=False): if D == 1: _, indices = pf.knn_indices_general(qrs, pts, K, True) else: _, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') if with_X_transformation: ######################## X-transformation ######################### X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def xconv(pts, fts, qrs, tag, N, K, D, P, C, depth_multiplier, is_training, C_pts_fts, with_x_transformation=True, sorting_method=None, with_global=False): ###xconv(pts,fts,qrs,tag,N,K,D,P,C,C_pts_fts,is_training,with_X_transformation,depth_multipiler,sorting_method,with_global) # @params: # N: Number of output points # K: Number of nearest neighbot # D: dilation rate # C: Number of output channels # P:the representative point number in the output, -1 means all input points are output representative points # x_transformation: replace max_pooling in PointNet # sorting_method: # with_global # pts:Input point cloud # qrs:queries # fts:features _, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') #(N,P,K,3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') #(N,P,1,3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') #(N,P,K,3) #Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') #X_transformation if with_x_transformation: ############################X_transformation####################################################### X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ##################################################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method=None, with_global=False): _, indices_dilated = pf.knn_indices_general( qrs, pts, K * D, True ) # 带孔K*D近邻; (N, P, 3) + (N, P, 3) => (N, P, K*D, 2) N=batch_num; P=point_num; 最后一维两个数:batch_indice + point_indice indices = indices_dilated[:, :, :: D, :] # (N, P, K, 2) 隔D取样,增加receptive field if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd( pts, indices, name=tag + 'nn_pts' ) # (N, P, 3) + (N, P, K, 2) => (N, P, K, 3) indices维度比pts高,最后一维做索引,batch_indice->N, point_indice->P, 最后剩下(N, P, 3)里的3是坐标,补进去返回形状是 (N, P, K, 3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) nn_pts_local = tf.subtract( nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3),减去中心点的坐标系,得到局部坐标系 # Prepare features to be transformed # 2个全连接层 对点的相对坐标系升维,(N, P, K, 3) => (N, P, K, C_pts_fts) # 升维的目的是为了和前面的特征拼接 (ct.) nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) # 如果第一次没有feature,就只使用从局部得到的点特征,如果有之前的特征,就拼接二者(ct.) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') # 拼接坐标特征和之前的特征 if with_X_transformation: ######################## X-transformation ######################### # (N, P, K, 3) > (N, P, 1, K * K) ; kernel size 是(1, K) X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) # (N, P, 1, K * K) > (N, P, K, K) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') # (N, P, K, K) > (N, P, 1, K * K) 可能为了较少参数量用depthwise_conv2d换conv2d X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) # (N, P, 1, K * K) > (N, P, K, K) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') # (N, P, K, K) > (N, P, 1, K * K) X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) # (N, P, 1, K * K) > (N, P, K, K) 最后的x矩阵 (K*K) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') # 用得到的 x 矩阵乘以之前的特征,所以这里其实是有2个分支,一个计算x矩阵,另外一个计算特征;(ct.) # (N, P, K, K) * (N, P, K, C_pts_fts) = (N, P, K, C_pts_fts) fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input # 最后的分离卷积 fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier ) # (N, P, K, C_pts_fts) -> (N, P, 1, C) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') # (N, P, C) # 用代表点全局位置信息, if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense( fts_global_0, C // 4, tag + 'fts_global', is_training ) # (N, P, C//4) 最后一层时作者使receptive field < 1让model更会捕捉local feature; 从“subvolume supervision”得到启发 to further address the over-fitting problem.; return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def xconv_cov(cov, pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method=None, with_global=False): """ cov: point covariance (B, N, 9) of previous layer pts: points of previous layer, fts: point features of previous layer, qrs: selected representative points of this layer, N: batch_size, K: neighbor_size, D: dilation parameter, P: the number of selected representative points, C: output feature number per point, C_pts_fts: feature number for each local point """ if D == 1: _, indices = pf.knn_indices_general(qrs, pts, K, True) else: _, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_covs = tf.gather_nd(cov, indices, name=tag + 'nn_pts') # (N, P, K, 9) # nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) # nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_covs, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training, with_bn=False) # following paper nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training, with_bn=False) # following paper if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') if with_X_transformation: ######################## X-transformation ######################### X_0 = pf.conv2d(nn_covs, K * K, tag + 'X_0', is_training, (1, K), with_bn=False) # following paper X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K), with_bn=False) # following paper X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), with_bn=False, activation=None) # following paper X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier, with_bn=True) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training, with_bn=True) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training, with_bn=True) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def deconv_new(pts, fts, qrs, tag, N, K, radius, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, D=1, sorting_method=None, with_global=False, knn=False): """ pts: points of previous layer, e.g.,(B, N/2, 3) fts: point features of previous layer, e.g.,(B, N/2, C) qrs: selected representative points of this layer, e.g.,(B, N, 3) N: batch_size, K: neighbor_size, D: dilation parameter, P: the number of selected representative points, C: output feature number per point, C_pts_fts: feature number for each local point radius: float32, the radius of query ball search knn: True: knn; False: query ball """ dist, idx = three_nn(qrs, pts) dist = tf.maximum(dist, 1e-10) norm = tf.reduce_sum((1.0 / dist), axis=2, keep_dims=True) norm = tf.tile(norm, [1, 1, 3]) weight = (1.0 / dist) / norm interpolated_fts = three_interpolate(fts, idx, weight) # (B, N, C) if knn: _, indices_dilated = pf.knn_indices_general(qrs, qrs, K * D, True) indices = indices_dilated[:, :, ::D, :] nn_pts = tf.gather_nd(qrs, indices, name=tag + 'nn_pts') # (B, N, K, 3) nn_fts_from_prev = tf.gather_nd(interpolated_fts, indices, name=tag + 'nn_fts') else: indices, pts_cnt = query_ball_point(radius, K, qrs, qrs) nn_pts = group_point(qrs, indices) nn_fts_from_prev = group_point(interpolated_fts, indices) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (B, N, 1, 3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (B, N, K, 3) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') if with_X_transformation: ######################## X-transformation ######################### X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) # following paper X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) # following paper X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) # following paper X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def ficonv(pts, fts, qrs, tag, N, K1, mm, sigma, scale, K, D, P, C, C_pts_fts, kernel_num, is_training, with_kernel_registering, with_kernel_shape_comparison, with_point_transformation, with_feature_transformation, with_learning_feature_transformation, kenel_initialization_method, depth_multiplier, sorting_method=None, with_global=False): Dis, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3) if with_point_transformation or with_feature_transformation: X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') if with_point_transformation: if with_learning_feature_transformation: nn_pts_local = tf.matmul(X_2_KK, nn_pts_local) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) else: # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) nn_pts_local = tf.matmul(X_2_KK, nn_pts_local) else: if with_learning_feature_transformation: nn_pts_local_ = tf.matmul(X_2_KK, nn_pts_local, name=tag + 'nn_pts_local_') # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local_, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) else: nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') P1 = tf.shape(nn_pts_local)[1] dim1 = 3 if with_kernel_registering: ######################## preparing ######################### if with_feature_transformation: nn_fts_input = tf.matmul(X_2_KK, nn_fts_input) r_data = tf.reduce_sum(nn_pts_local * nn_pts_local, axis=3, keep_dims=True, name=tag + 'kernel_pow') ######################## kernel-registering ######################### shape_id = 0 if kenel_initialization_method == 'random': kernel_shape = tf.Variable(tf.random_uniform([K1, dim1], minval=-0.5, maxval=0.5, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) else: kernel_shape = tf.Variable(tf.random_normal([K1, dim1], mean=0.0, stddev=1.0, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) kernel_shape_dis = tf.sqrt(tf.reduce_sum(kernel_shape * kernel_shape, axis=1), name=tag + 'kernel_shape_dis' + str(shape_id)) kernel_shape_normal = scale * tf.div( kernel_shape, tf.reduce_max(kernel_shape_dis), name=tag + 'kernel_shape_normal' + str(shape_id)) r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal, axis=1, keep_dims=True, name=tag + 'kernel_pow' + str(shape_id)) reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1], name=tag + 'reshape_kernel' + str(shape_id)) m = tf.reshape(tf.matmul(reshape_data, tf.transpose(kernel_shape_normal)), [N, P1, K, K1], name=tag + 'mm' + str(shape_id)) dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel), perm=[0, 1, 3, 2], name=tag + 'dis_matrix' + str(shape_id)) coef_matrix = tf.exp(tf.div(-dis_matrix, sigma), name=tag + 'coef_matrix' + str(shape_id)) #coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id)) if with_kernel_shape_comparison: coef_global = tf.reduce_sum( coef_matrix, axis=[2, 3], keep_dims=True) / K coef_normal = coef_global * tf.div( coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) else: coef_normal = tf.div(coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) fts_X = tf.matmul(coef_normal, nn_fts_input, name=tag + 'fts_X' + str(shape_id)) ################################################################### fts_conv = pf.separable_conv2d(fts_X, math.ceil(mm * C / kernel_num), tag + 'fts_conv' + str(shape_id), is_training, (1, K1), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d' + str(shape_id)) for shape_id in range(kernel_num - 1): shape_id = shape_id + 1 if kenel_initialization_method == 'random': kernel_shape = tf.Variable(tf.random_uniform([K1, dim1], minval=-0.5, maxval=0.5, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) else: kernel_shape = tf.Variable(tf.random_normal([K1, dim1], mean=0.0, stddev=1.0, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) kernel_shape_dis = tf.sqrt( tf.reduce_sum(kernel_shape * kernel_shape, axis=1), name=tag + 'kernel_shape_dis' + str(shape_id)) kernel_shape_normal = scale * tf.div( kernel_shape, tf.reduce_max(kernel_shape_dis), name=tag + 'kernel_shape_normal' + str(shape_id)) r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal, axis=1, keep_dims=True, name=tag + 'kernel_pow' + str(shape_id)) reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1], name=tag + 'reshape_kernel' + str(shape_id)) m = tf.reshape(tf.matmul(reshape_data, tf.transpose(kernel_shape_normal)), [N, P1, K, K1], name=tag + 'mm' + str(shape_id)) dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel), perm=[0, 1, 3, 2], name=tag + 'dis_matrix' + str(shape_id)) coef_matrix = tf.exp(tf.div(-dis_matrix, sigma), name=tag + 'coef_matrix' + str(shape_id)) #coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id)) if with_kernel_shape_comparison: coef_global = tf.reduce_sum( coef_matrix, axis=[2, 3], keep_dims=True) / K coef_normal = coef_global * tf.div( coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) else: coef_normal = tf.div(coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) fts_X = tf.matmul(coef_normal, nn_fts_input, name=tag + 'fts_X' + str(shape_id)) ################################################################### fts_conv = pf.separable_conv2d(fts_X, math.ceil(mm * C / kernel_num), tag + 'fts_conv' + str(shape_id), is_training, (1, K1), depth_multiplier=depth_multiplier) fts_conv_3d = tf.concat( [fts_conv_3d, tf.squeeze(fts_conv, axis=2)], axis=-1, name=tag + 'fts_conv_3d' + str(shape_id)) else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method=None, with_global=False): ''' pts: batch points input fts: batch feature inputs qrs: points outputs tag: tags N: batch_size K: k neighbors, like kernel size in normal conv2d D: dilation rate P: sample points size in one sample C: input channel num C_pts_fts: output channel num ''' # get k*d near points'indices from pts according to qrs _, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) # select k points' indices indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) # get the k points' data according to the indices nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3) # expand qrs from (N, P, 3) to (N, P, 1, 3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) # remove qrs from pts nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3) # Prepare features to be transformed # step 2, MLP nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) # step 3, concat origin features with the new features if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') # step 4, according to the appendix, the second MLP is formes with two depthwise_conv layers if with_X_transformation: ######################## X-transformation ######################### X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') # step 5 fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input # step 6 fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training, with_bn=False) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training, with_bn=False) #fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) #fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d