def input_transform_net(points, is_training, name, dim=3, reused=None): """ Input (XYZ) Transform Net, input is BxNx3 points data Return: Transformation matrix of size 3xK """ pts_num = points.get_shape()[1].value batch_size = points.get_shape()[0].value net = pf.dense(points, 64, name + 'mlp1', is_training, activation_fn=tf.nn.relu, reuse=reused) net = pf.dense(net, 128, name + 'mlp2', is_training, activation_fn=tf.nn.relu, reuse=reused) net = pf.dense(net, 1024, name + 'mlp3', is_training, activation_fn=tf.nn.relu, reuse=reused) net = tf.expand_dims(net, axis=-1, name=name+'expand_dim') net = pf.max_pool2d(net, [pts_num, 1], stride=[1, 1], name=name + 'pooling') net = tf.reshape(net, [batch_size, 1024]) net = pf.dense(net, 128, name + 'mlp4', is_training, activation_fn=tf.nn.relu, reuse=reused) net = pf.dense(net, 64, name + 'mlp5', is_training, activation_fn=tf.nn.relu, reuse=reused) with tf.variable_scope(name + 'xyz_transform', reuse=reused): assert(dim == 3) weights = tf.get_variable('weights', [64, 3*dim], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [3*dim], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([1, 0, 0, 0, 1, 0, 0, 0, 1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 3, dim]) return transform
def __call__(self, point_cloud): ''' input: point_cloud: BxNx3 output: latent_code: Bxlatent_code_dim ''' with tf.variable_scope(self.name): #batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz #sort_mtd = 'cxyz' sort_mtd= None # Set Abstraction layers l1_xyz, l1_points, l1_indices = point_conv_module(l0_xyz, l0_points, npoint=512, c_fts_out=128, k_neighbors=64, d_rate=1, is_training=is_training, scope='conv_layer_1', sorting_method=sort_mtd) l2_xyz, l2_points, l2_indices = point_conv_module(l1_xyz, l1_points, npoint=128, c_fts_out=256, k_neighbors=32, d_rate=1, is_training=is_training, scope='conv_layer_2', sorting_method=sort_mtd) l3_xyz, l3_points, l3_indices = point_conv_module(l2_xyz, l2_points, npoint=24, c_fts_out=512, k_neighbors=8, d_rate=1, is_training=is_training, scope='conv_layer_3', sorting_method=sort_mtd) # fc fc = pf.dense(l5_points, 1024, 'fc1', self.is_training) fc = pf.dense(fc, 512, 'fc2', self.is_training) latent_code = pf.dense(fc, self.latent_code_dim, 'fc3', self.is_training) # B x latent_code_dim return latent_code
def input_qtransform_net(points, is_training, name, reused=None): batch_size = points.get_shape()[0].value point_num = points.get_shape()[1].value net = pf.dense(points, 64, name + 'mlp1', is_training, activation_fn=tf.nn.relu, reuse=reused) net = pf.dense(net, 128, name + 'mlp2', is_training, activation_fn=tf.nn.relu, reuse=reused) net = pf.dense(net, 1024, name + 'mlp3', is_training, activation_fn=tf.nn.relu, reuse=reused) net = tf.expand_dims(net, axis=-1, name=name + 'expand_dim') net = pf.max_pool2d(net, [point_num, 1], stride=[1, 1], name=name+'pooling') net = tf.reshape(net, [batch_size, 1, 1024]) net = pf.dense(net, 128, name + 'mlp4', is_training, activation_fn=tf.nn.relu, reuse=reused) net = pf.dense(net, 64, name + 'mlp5', is_training, activation_fn=tf.nn.relu, reuse=reused) with tf.variable_scope(name+'q_transform', reuse=reused): weights = tf.get_variable('weights', [64, 4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([1, 0, 0, 0], dtype=tf.float32) weights = tf.expand_dims(weights, 0) weights = tf.tile(weights, [batch_size, 1, 1]) transform = tf.matmul(net, weights) # [batch_size, 1, 4] each shape has a trans matrix transform = tf.nn.bias_add(transform, biases) qtransform = batch_quat_to_rotmat(transform) return qtransform
def seg_shape(self, feats, parts_num, name, is_training, is_resued=None): net = pf.dense(feats, 1024, name + 'mlp1', is_training, reuse=is_resued, activation_fn=tf.nn.relu) net = pf.dense(net, 512, name + 'mlp2', is_training, reuse=is_resued, activation_fn=tf.nn.relu) net = pf.dense(net, 256, name + 'mlp3', is_training, reuse=is_resued, activation_fn=tf.nn.relu) net = pf.dense(net, 128, name + 'mlp4', is_training, reuse=is_resued, activation_fn=tf.nn.relu) net = pf.dense(net, parts_num, name + 'mlp5', is_training, reuse=is_resued, activation_fn=None) return net
def points_category(self, shape_fts, category_num, is_training, name, is_resused=None): net = pf.dense(shape_fts, 512, name + 'mlp1', is_training, reuse=is_resused, activation_fn=tf.nn.relu) net = tf.layers.dropout(net, 0.3, training=is_training, name=name + 'dp1') net = pf.dense(net, 256, name + 'mlp2', is_training, reuse=is_resused, activation_fn=tf.nn.relu) net = tf.layers.dropout(net, 0.3, training=is_training, name=name + 'dp2') net = pf.dense(net, category_num, name + 'mlp3', is_training, reuse=is_resused, activation_fn=None) return net
def points_deform(self, feats, name, is_training, is_reused=None): bottleneck_size = feats.get_shape()[-1].value feats = pf.dense(feats, bottleneck_size, name + 'mlp1', is_training, reuse=is_reused, activation_fn=tf.nn.relu) feats = pf.dense(feats, bottleneck_size // 2, name + 'mlp2', is_training, reuse=is_reused, activation_fn=tf.nn.relu) feats = pf.dense(feats, bottleneck_size // 4, name + 'mlp3', is_training, reuse=is_reused, activation_fn=tf.nn.relu) feats = pf.dense(feats, 3, name + 'mlp4', is_training, reuse=is_reused, activation_fn=tf.nn.tanh) points_deform = 2 * feats # why ??? return points_deform
def input_transform_net(point_cloud, is_training,N, PP,KK=3): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ input_image = tf.expand_dims(point_cloud, -1) print(input_image) net = pf.conv2d(input_image, 64, 'tconv1', is_training,(1,3)) net = pf.dense(net, 128, 'tconv2', is_training) net = pf.dense(net, 1024, 'tconv3', is_training) print(net) net = pf.max_pool2d(net,[1024,1],'tmaxpool') net = pf.dense(net, 512, 'tfc1',is_training) net = pf.dense(net, 256, 'tfc2',is_training) net = tf.reshape(net, (N, -1)) with tf.variable_scope('transform_XYZ') as sc: assert(KK==3) weights = tf.get_variable('weights', (256, 3*KK), initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', (3*KK), initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([1,0,0,0,1,0,0,0,1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, (N, 3, KK)) return transform
def point_feat_extract(self, points, name, is_training, is_trans, is_reused=None): pts_num = points.get_shape()[1].value if is_trans == 'q_trans': transform_matrix = input_qtransform_net(points, is_training, name+'rotation', reused = is_reused) points_t = tf.matmul(points, transform_matrix) else: if is_trans == 'trans': transform_matrix = input_transform_net(points, is_training, name+'rotation', reused = is_reused) points_t = tf.matmul(points, transform_matrix) else: points_t = points net = pf.dense(points_t, 64, name+'mlp1', is_training, reuse=is_reused, activation_fn=tf.nn.relu) net = pf.dense(net, 128, name+'mlp2', is_training, reuse=is_reused, activation_fn=tf.nn.relu) net = pf.dense(net, 256, name+'mlp3', is_training, reuse=is_reused, activation_fn=tf.nn.relu) net = pf.dense(net, 1024, name+'mlp4', is_training, reuse=is_reused, activation_fn=tf.nn.relu) pfts = net net = tf.expand_dims(net, axis=-1, name=name+'extend_input') net = pf.max_pool2d(net, [pts_num, 1], stride=[1, 1], name=name+'pooling') net_tile = tf.tile(tf.squeeze(net, axis=-1), [1, pts_num, 1]) concat_pfts = tf.concat([pfts, net_tile], axis=-1) if is_trans is not 'no_trans': return concat_pfts, tf.squeeze(net, axis=[1, 3]), transform_matrix, points_t else: return concat_pfts, tf.squeeze(net, axis=[1, 3]), points_t
def pointnet1(point_cloud, is_training, N, PP): transform = input_transform_net(point_cloud, is_training, N, PP, KK=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) # (N,PP,3,1) net = pf.conv2d(input_image, 64, 'p1_conv1', is_training,(1,3)) # (N,PP,1,64) net = pf.dense(net, 64, 'p1_mlp1', is_training) # (N,PP,1,64) net = pf.dense(net, 64, 'p1_mlp2', is_training) # (N,PP,1,64) net = pf.dense(net, 128, 'p1_mlp3', is_training) # (N,PP,1,128) net = pf.dense(net, 1024, 'p1_mlp4', is_training) # (N,PP,1,1024) net1 = net net = pf.max_pool2d(net,[PP,1],'p1_maxpool') # (N,1,1,1024) net = tf.squeeze(net, axis=[2]) net1 = tf.squeeze(net1, axis=[2]) return net,net1
def __init__(self, points, features, is_training, setting): PointCNN.__init__(self, points, features, is_training, setting) self.point_features = self.layer_fts[1] fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keepdims=True, name='fc_mean') self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean) self.logits = pf.dense(self.fc_layers[-1], setting.num_class, 'logits', is_training, with_bn=False, activation=None)
def __init__(self, points, features, is_training, setting): PointCNN.__init__(self, points, features, is_training, setting) self.logits = pf.dense(self.fc_layers[-1], setting.num_class, 'logits', is_training, with_bn=False, activation=None)
def __init__(self, points, features, is_training, setting): PointCNN.__init__(self, points, features, is_training, setting) with tf.variable_scope(setting.network_name): batch_size = points.get_shape()[0].value fc_flatten = tf.reshape(self.fc_layers[-1], [batch_size, -1]) fc_flatten = tf.concat([fc_flatten, features[:, 0, :]], axis=1) fc1 = pf.dense(fc_flatten, 512, 'extra_fc_1', is_training) # fc1_drop = tf.layers.dropout(fc1, 0.0, training=is_training, name='extra_fc_1_drop') # self.fc_layers.append(fc1_drop) fc2 = pf.dense(fc1, 256, 'extra_fc_2', is_training) self.output = pf.dense(fc2, 3 + NUM_HEADING_BIN * 2 + NUM_SIZE_CLUSTER * 4, 'output', is_training, with_bn=False, activation=None)
def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method=None, with_global=False): if D == 1: _, indices = pf.knn_indices_general(qrs, pts, K, True) else: _, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') if with_X_transformation: ######################## X-transformation ######################### X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def point_conv_transpose_fc_no_skip(pts_in, fts_in, indices, np_out, K, C, is_training, activation=tf.nn.relu, bn=True, tag='point_deconv'): ''' NOTE: no skipped link of output points and features pts_in: xyz points, Tensor, (batch_size, npoint, 3) fts_in: features, Tensor, (batch_size, npoint, channel_fts) np_out: number of points that is expected K: neighbor size C: output feature channel number ''' with tf.variable_scope(tag): batch_size = fts_in.shape[0] n_pts_in = fts_in.shape[1] if pts_in is None: nn_fts_input = fts_in # (N, P, C_fts_in) #print('Error: fts_in is None.') else: nn_fts_input = tf.concat( [pts_in, fts_in], axis=-1, name='_nn_fts_input') # (N, P, C_fts_in+3) nn_fts_input = tf.expand_dims(nn_fts_input, axis=2) # (N, P, 1, C_fts_in) # using fc fts_fc = pf.dense(nn_fts_input, K * C, 'fc_fts_prop', is_training) # (N, P, 1, K*C) fts_fc = tf.reshape(fts_fc, (batch_size, n_pts_in, K, C)) # (N, P, K, C) fts_pts_conv = pf.conv2d( fts_all_concat, C, tag + '_fts_pts_conv1D', is_training, kernel_size=(1, 1), strides=(1, 1), with_bn=bn, activation=activation) # (N, P, K, C+3_ch_skip?) -> (N, P, K, C) # summing up feature for each point fts_dense = tf.scatter_nd( tf.reshape(indices, (-1, 2)), tf.reshape(fts_pts_conv, (-1, C)), (batch_size, n_pts_out, C)) # (N, N_PTS_OUT, c) return fts_dense
def __init__(self, points, features, is_training, setting): PointCNN_SEG.__init__(self, points, features, is_training, setting) ###Classification self.classification_logits = pf.dense( self.fc_layers_classification[-1], setting.num_class, 'logits_classification', is_training, with_bn=False, activation=None) ###Segmentation Mask self.segmentation_logits = pf.dense(self.fc_layers_segmentation[-1], 2, 'logits_segmentation', is_training, with_bn=False, activation=None)
def pointcnn_xupconv_module(xyz, points, dense_xyz, dense_points_skip, c_fts_out, c_x, k_neighbors, d_rate, is_training, depth_multiplier, with_global, scope): ''' input: xyz: TF tensor, input point clouds coords, B x N x 3 points: TF tensor, input point clouds features, B x N x fts_channel dense_xyz: TF tensor, representative points previously sampled, B x npoint x 3 dense_points_skip: skip link from previously generated features c_fts_out: int32, output channels number c_x: int32, channels number of lifted features for x-transformation matrix k_neighbors: int32, neighbor size d_rate: int32, dilation rate is_training: TF tensor, flag indicating training status output: new_xyz: TF tensor, output point clouds coords, B x npoint x 3 new_points: TF tensor, output point clouds features, B x npoint x c_fts_out ''' with tf.variable_scope(scope): new_xyz = dense_xyz npoint = new_xyz.shape[1] new_points = xconv(xyz, points, new_xyz, scope, xyz.shape[0], k_neighbors, d_rate, npoint, c_fts_out, c_x, is_training, True, depth_multiplier, sorting_method=None, with_global=with_global) if dense_points_skip is not None: points_concat = tf.concat([new_points, dense_points_skip], axis=-1, name=scope + '_skip_concat') new_points = pf.dense(points_concat, c_fts_out, scope + '_fuse', is_training) return new_xyz, new_points
def __init__(self, points, features, is_training, setting): PointCNN.__init__(self, points, features, is_training, setting) fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keep_dims=True, name='fc_mean') self.feature_list = tf.reshape(self.feature_list, [128, 61440]) self.feature_list_A = self.feature_list[0:64] self.feature_list_B = self.feature_list[64:128] self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean) #最后一层连接 self.logits = pf.dense(self.fc_layers[-1], setting.num_class, 'logits', is_training, with_bn=False, activation=None)
def get_model(layer_pts, is_training, RIconv_params, RIdconv_params, fc_params, sampling='fps', weight_decay=0.0, bn_decay=None, part_num=50): if sampling == 'fps': sys.path.append(os.path.join(BASE_DIR, 'tf_ops/sampling')) from tf_sampling import farthest_point_sample, gather_point layer_fts_list = [None] layer_pts_list = [layer_pts] for layer_idx, layer_param in enumerate(RIconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] P = layer_param['P'] C = layer_param['C'] # qrs = layer_pts if P == -1 else layer_pts[:,:P,:] # (N, P, 3) if P == -1: qrs = layer_pts else: if sampling == 'fps': qrs = gather_point(layer_pts, farthest_point_sample(P, layer_pts)) elif sampling == 'random': qrs = tf.slice(layer_pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() layer_fts= RIConv(layer_pts_list[-1], layer_fts_list[-1], qrs, is_training, tag, K, D, P, C, True, bn_decay) layer_pts = qrs layer_pts_list.append(qrs) layer_fts_list.append(layer_fts) if RIdconv_params is not None: fts = layer_fts_list[-1] for layer_idx, layer_param in enumerate(RIdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] # 2 1 0 qrs_layer_idx = layer_param['qrs_layer_idx'] # 1 0 -1 pts = layer_pts_list[pts_layer_idx + 1] qrs = layer_pts_list[qrs_layer_idx + 1] fts_qrs = layer_fts_list[qrs_layer_idx + 1] C = fts_qrs.get_shape()[-1].value if fts_qrs is not None else C//2 P = qrs.get_shape()[1].value layer_fts= RIConv(pts, fts, qrs, is_training, tag, K, D, P, C, True, bn_decay) if fts_qrs is not None: # this is for last layer fts_concat = tf.concat([layer_fts, fts_qrs], axis=-1, name=tag + 'fts_concat') fts = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) else: fts = layer_fts for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] layer_fts = pf.dense(layer_fts, C, 'fc{:d}'.format(layer_idx), is_training) layer_fts = tf.layers.dropout(layer_fts, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) logits_seg = pf.dense(layer_fts, part_num, 'logits', is_training, with_bn=False, activation=None) return logits_seg
def __init__(self, points, features, num_class, is_training, setting, task): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.with_fps: from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: C_fts = xconv_params[0][-1] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K, D, P, C = layer_param # get k-nearest points pts = self.layer_pts[-1] fts = self.layer_fts[-1] if P == -1: qrs = points else: if setting.with_fps: qrs = tf_sampling.gather_point( pts, tf_sampling.farthest_point_sample(P, pts)) # (N,P,3) else: qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) self.layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1][-1] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) self.layer_fts.append(fts_xconv) if task == 'segmentation': for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K, D, pts_layer_idx, qrs_layer_idx = layer_param pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[ pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] _, _, P, C = xconv_params[qrs_layer_idx] _, _, _, C_prev = xconv_params[pts_layer_idx] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): channel_num, drop_rate = layer_param fc = pf.dense(self.fc_layers[-1], channel_num, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, drop_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop) logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None) if task == 'classification': logits_mean = tf.reduce_mean(logits, axis=1, keep_dims=True, name='logits_mean') self.logits = tf.cond(is_training, lambda: logits, lambda: logits_mean) elif task == 'segmentation': self.logits = logits else: print('Unknown task!') exit() self.probs = tf.nn.softmax(self.logits, name='probs')
def __init__(self, points, features, is_training, setting): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.sampling == 'fps': from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: # expand feature dimentions features = tf.reshape(features, (N, -1, setting.data_dim - 3), name='features_reshape') C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] # neighborhood size D = layer_param['D'] # dilate rate P = layer_param['P'] # representative point number C = layer_param['C'] # output channel number links = layer_param[ 'links'] # e.g. [-1,-2] will tell the current layer to receive inputs from the previous two layers if setting.sampling != 'random' and links: print( 'Error: flexible links are supported only when random sampling is used!' ) exit() # get k-nearest points pts = self.layer_pts[-1] # get the end one in list (last layer) fts = self.layer_fts[-1] # if P == -1 or p equals to last layer's p , than no sampling if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = self.layer_pts[-1] else: # for segmentation task if setting.sampling == 'fps': # fps_indices: [[0_0, 0_1, ...., 0_P-1], ...., [N-1_0, N-1_1, ...., N-1_P-1]] (NxP) fps_indices = tf_sampling.farthest_point_sample(P, pts) # replicating a index(0...N) for each output poit(P) -> (N, P, 1) # batch_indices: [[[0],[0],...,[0]], ..., [[N-1],[N-1],...,[N-1]]] batch_indices = tf.tile( tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1)) # tf.expand_dims(fps_indices, -1): [[[0_0], [0_1], ..., [0_P-1]], ... , [[N-1_0], [N-1_1], ... , [N-1_P-1]]] # indices: [[[0, 0_0], [0, 0_1], ..., [0, 0_P-1]], ..., [[N-1, N-1_0], [N-1, N-1_1], ... , [N-1, N-1_P-1]]] indices = tf.concat( [batch_indices, tf.expand_dims(fps_indices, -1)], axis=-1) qrs = tf.gather_nd(pts, indices, name=tag + 'qrs') # (N, P, 3) elif setting.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) # for classification tasks elif setting.sampling == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() self.layer_pts.append(qrs) if layer_idx == 0: # lift each point coordinates into C_pts_fts dimensional space C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) # with global feature at last layer with_global = (setting.with_global and layer_idx == len(xconv_params) - 1) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method, with_global) fts_list = [] # receive inputs from previous layers for link in links: fts_from_link = self.layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice_' + str(-link)) fts_list.append(fts_slice) if fts_list: fts_list.append(fts_xconv) self.layer_fts.append( tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: self.layer_fts.append(fts_xconv) if hasattr(setting, 'xdconv_params'): for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[ pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop)
def ficonv(pts, fts, qrs, tag, N, K1, mm, sigma, scale, K, D, P, C, C_pts_fts, kernel_num, is_training, with_kernel_registering, with_kernel_shape_comparison, with_point_transformation, with_feature_transformation, with_learning_feature_transformation, kenel_initialization_method, depth_multiplier, sorting_method=None, with_global=False): Dis, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3) if with_point_transformation or with_feature_transformation: X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') if with_point_transformation: if with_learning_feature_transformation: nn_pts_local = tf.matmul(X_2_KK, nn_pts_local) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) else: # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) nn_pts_local = tf.matmul(X_2_KK, nn_pts_local) else: if with_learning_feature_transformation: nn_pts_local_ = tf.matmul(X_2_KK, nn_pts_local, name=tag + 'nn_pts_local_') # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local_, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) else: nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') P1 = tf.shape(nn_pts_local)[1] dim1 = 3 if with_kernel_registering: ######################## preparing ######################### if with_feature_transformation: nn_fts_input = tf.matmul(X_2_KK, nn_fts_input) r_data = tf.reduce_sum(nn_pts_local * nn_pts_local, axis=3, keep_dims=True, name=tag + 'kernel_pow') ######################## kernel-registering ######################### shape_id = 0 if kenel_initialization_method == 'random': kernel_shape = tf.Variable(tf.random_uniform([K1, dim1], minval=-0.5, maxval=0.5, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) else: kernel_shape = tf.Variable(tf.random_normal([K1, dim1], mean=0.0, stddev=1.0, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) kernel_shape_dis = tf.sqrt(tf.reduce_sum(kernel_shape * kernel_shape, axis=1), name=tag + 'kernel_shape_dis' + str(shape_id)) kernel_shape_normal = scale * tf.div( kernel_shape, tf.reduce_max(kernel_shape_dis), name=tag + 'kernel_shape_normal' + str(shape_id)) r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal, axis=1, keep_dims=True, name=tag + 'kernel_pow' + str(shape_id)) reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1], name=tag + 'reshape_kernel' + str(shape_id)) m = tf.reshape(tf.matmul(reshape_data, tf.transpose(kernel_shape_normal)), [N, P1, K, K1], name=tag + 'mm' + str(shape_id)) dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel), perm=[0, 1, 3, 2], name=tag + 'dis_matrix' + str(shape_id)) coef_matrix = tf.exp(tf.div(-dis_matrix, sigma), name=tag + 'coef_matrix' + str(shape_id)) #coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id)) if with_kernel_shape_comparison: coef_global = tf.reduce_sum( coef_matrix, axis=[2, 3], keep_dims=True) / K coef_normal = coef_global * tf.div( coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) else: coef_normal = tf.div(coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) fts_X = tf.matmul(coef_normal, nn_fts_input, name=tag + 'fts_X' + str(shape_id)) ################################################################### fts_conv = pf.separable_conv2d(fts_X, math.ceil(mm * C / kernel_num), tag + 'fts_conv' + str(shape_id), is_training, (1, K1), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d' + str(shape_id)) for shape_id in range(kernel_num - 1): shape_id = shape_id + 1 if kenel_initialization_method == 'random': kernel_shape = tf.Variable(tf.random_uniform([K1, dim1], minval=-0.5, maxval=0.5, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) else: kernel_shape = tf.Variable(tf.random_normal([K1, dim1], mean=0.0, stddev=1.0, dtype=tf.float32), name=tag + 'kernel_shape' + str(shape_id)) kernel_shape_dis = tf.sqrt( tf.reduce_sum(kernel_shape * kernel_shape, axis=1), name=tag + 'kernel_shape_dis' + str(shape_id)) kernel_shape_normal = scale * tf.div( kernel_shape, tf.reduce_max(kernel_shape_dis), name=tag + 'kernel_shape_normal' + str(shape_id)) r_kernel = tf.reduce_sum(kernel_shape_normal * kernel_shape_normal, axis=1, keep_dims=True, name=tag + 'kernel_pow' + str(shape_id)) reshape_data = tf.reshape(nn_pts_local, [N * P1 * K, dim1], name=tag + 'reshape_kernel' + str(shape_id)) m = tf.reshape(tf.matmul(reshape_data, tf.transpose(kernel_shape_normal)), [N, P1, K, K1], name=tag + 'mm' + str(shape_id)) dis_matrix = tf.transpose(r_data - 2 * m + tf.transpose(r_kernel), perm=[0, 1, 3, 2], name=tag + 'dis_matrix' + str(shape_id)) coef_matrix = tf.exp(tf.div(-dis_matrix, sigma), name=tag + 'coef_matrix' + str(shape_id)) #coef_matrix = tf.transpose(r_data-2*m+tf.transpose(r_kernel),perm=[0,1,3,2],name=tag+'coef_matrix'+str(shape_id)) if with_kernel_shape_comparison: coef_global = tf.reduce_sum( coef_matrix, axis=[2, 3], keep_dims=True) / K coef_normal = coef_global * tf.div( coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) else: coef_normal = tf.div(coef_matrix, tf.reduce_sum(coef_matrix, axis=3, keep_dims=True), name=tag + 'coef_normal' + str(shape_id)) fts_X = tf.matmul(coef_normal, nn_fts_input, name=tag + 'fts_X' + str(shape_id)) ################################################################### fts_conv = pf.separable_conv2d(fts_X, math.ceil(mm * C / kernel_num), tag + 'fts_conv' + str(shape_id), is_training, (1, K1), depth_multiplier=depth_multiplier) fts_conv_3d = tf.concat( [fts_conv_3d, tf.squeeze(fts_conv, axis=2)], axis=-1, name=tag + 'fts_conv_3d' + str(shape_id)) else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def __init__(self, points, features, is_training, setting): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.sampling == 'fps': import sys #sys.path.append("/home/hasan/data/paper2/PointCNN/") from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: features = tf.reshape(features, (N, -1, setting.data_dim - 3), name='features_reshape') C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] P = layer_param['P'] C = layer_param['C'] links = layer_param['links'] if setting.sampling != 'random' and links: print( 'Error: flexible links are supported only when random sampling is used!' ) exit() # get k-nearest points pts = self.layer_pts[-1] fts = self.layer_fts[-1] if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = self.layer_pts[-1] else: if setting.sampling == 'fps': fps_indices = tf_sampling.farthest_point_sample(P, pts) batch_indices = tf.tile( tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1)) indices = tf.concat( [batch_indices, tf.expand_dims(fps_indices, -1)], axis=-1) qrs = tf.gather_nd(pts, indices, name=tag + 'qrs') # (N, P, 3) elif setting.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) elif setting.sampling == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() self.layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) with_global = (setting.with_global and layer_idx == len(xconv_params) - 1) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method, with_global) fts_list = [] for link in links: fts_from_link = self.layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice_' + str(-link)) fts_list.append(fts_slice) if fts_list: fts_list.append(fts_xconv) self.layer_fts.append( tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: self.layer_fts.append(fts_xconv) if hasattr(setting, 'xdconv_params'): for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[ pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop)
def xconv(pts, fts, qrs, tag, N, K, D, P, C, depth_multiplier, is_training, C_pts_fts, with_x_transformation=True, sorting_method=None, with_global=False): ###xconv(pts,fts,qrs,tag,N,K,D,P,C,C_pts_fts,is_training,with_X_transformation,depth_multipiler,sorting_method,with_global) # @params: # N: Number of output points # K: Number of nearest neighbot # D: dilation rate # C: Number of output channels # P:the representative point number in the output, -1 means all input points are output representative points # x_transformation: replace max_pooling in PointNet # sorting_method: # with_global # pts:Input point cloud # qrs:queries # fts:features _, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') #(N,P,K,3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') #(N,P,1,3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') #(N,P,K,3) #Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') #X_transformation if with_x_transformation: ############################X_transformation####################################################### X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ##################################################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def Invariance_Transformation_Net(point_cloud, features, is_training, invarians_trans_param): xconv_params = invarians_trans_param.xconv_params fc_params = invarians_trans_param.fc_params with_X_transformation = invarians_trans_param.with_X_transformation sorting_method = invarians_trans_param.sorting_method #N=point_cloud.get_shape()[1].value N = point_cloud.get_shape()[0].value if invarians_trans_param.sampling == 'fps': import tf_sampling layer_pts = [point_cloud] if features is None: layer_fts = [features] else: features = tf.reshape(features, (N, -1, invarians_trans_param.data_dim - 3), name='features_reshape') C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str( layer_idx + 1) + '_' #####xconv_1_ #####xconv_2_ #####xconv_3_ #####xconv_4_ K = layer_param['K'] D = layer_param['D'] P = layer_param['P'] C = layer_param['C'] links = layer_param['links'] ## type(layer_param) is dict #get k-nearest points pts = layer_pts[-1] fts = layer_fts[-1] if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = layer_pts[-1] else: if invarians_trans_param.sampling == 'fps': fps_indices = tf_sampling.farthest_point_sample(P, pts) batch_indices = tf.tile(tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1)) indices = tf.concat( [batch_indices, tf.expand_dims(fps_indices, -1)], axis=-1) qrs = tf.gather_nd(pts, indices, name=tag + 'qrs') ### (N,P,3) print(tf.shape(qrs)) elif invarians_trans_param.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) elif invarians_trans_param == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') ### (N,P,3) else: print('unknown sampling method') exit() layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multipiler = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multipiler = math.ceil(C / C_prev) with_global = (invarians_trans_param.with_global and layer_idx == len(xconv_params) - 1) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multipiler, sorting_method, with_global) fts_list = [] for link in links: fts_from_link = layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice' + str(-link)) fts_list.append(fts_slice) if fts_list: fts_list.append(fts_xconv) layer_fts.append( tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: layer_fts.append(fts_xconv) if hasattr(invarians_trans_param, 'xdconv_params'): for layer_idx, layer_param in enumerate( invarians_trans_param.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = layer_pts[pts_layer_idx + 1] fts = layer_fts[pts_layer_idx + 1] if layer_idx == 0 else layer_fts[-1] qrs = layer_pts[qrs_layer_idx + 1] fts_qrs = layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multipiler = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multipiler, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) layer_pts.append(qrs) layer_fts.append(fts_fuse) fc_layers = layer_fts[-1] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) fc_layers.append(fc_drop) return fc_layers
def main(): num_class = 6 pts_fts = tf.placeholder(tf.float32, [2,1,7]) labels = tf.placeholder(tf.int32, [2,]) global_step = tf.Variable(0, trainable=False, name='global_step') is_training = tf.placeholder(tf.bool, name='is_training') features_hd = pts_fts#pf.dense(pts_fts, 16, 'features_hd', is_training) #fc = pf.dense(features_hd, 128, 'fc', is_training, with_bn=True) dense = tf.layers.dense(features_hd, units=16, activation=tf.nn.elu, kernel_initializer=tf.glorot_uniform_initializer(), kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=1.0)) fc = tf.layers.batch_normalization(dense, momentum=0.9, training=is_training, beta_regularizer=tf.contrib.layers.l2_regularizer(scale=1.0), gamma_regularizer=tf.contrib.layers.l2_regularizer(scale=1.0)) logits = pf.dense(fc, num_class, 'logits', is_training, with_bn=False, activation=None) probs = tf.nn.softmax(logits, name='probs') labels_2d = tf.expand_dims(labels, axis=-1, name='labels_2d') labels_tile = tf.tile(labels_2d, (1, tf.shape(probs)[1]), name='labels_tile') #TODO labels_tile = labels logits = tf.reduce_mean(logits, axis=1) #TODO loss_op = tf.losses.sparse_softmax_cross_entropy(labels=labels_tile, logits=logits) #loss_op = tf.nn.softmax_cross_entropy_with_logits(labels=labels_tile, logits=logits) lr_exp_op = tf.train.exponential_decay(0.001, global_step, 1000, 0.5, staircase=True) lr_clip_op = tf.maximum(lr_exp_op, 1e-6) reg_loss = 0.0 * tf.losses.get_regularization_loss() optimizer = tf.train.AdamOptimizer(learning_rate=lr_clip_op, epsilon=1e-2) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = optimizer.minimize(loss_op + reg_loss, global_step=global_step) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) gpu_options = tf.GPUOptions(allow_growth=True) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: sess.run(init_op) pts_fts_v = [np.array([[[2,3,5,1,6,4,8]], [[2,6,7,2,4,7,4]]]).astype(np.float32), \ np.array([[[2,6,7,2,4,7,4]], [[2,3,5,1,6,4,8]]]).astype(np.float32)] labels_v = [np.array([0,3]).astype(np.int32), np.array([3,0]).astype(np.int32)] for idx in range(10000): out_val = \ sess.run([dense, fc, train_op, loss_op], feed_dict={ pts_fts: pts_fts_v[idx%2], labels: labels_v[idx%2], is_training: True, }) print('dense') print(out_val[0]) print('fc') print(out_val[1]) print(out_val[-1])
def xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method=None, with_global=False): _, indices_dilated = pf.knn_indices_general( qrs, pts, K * D, True ) # 带孔K*D近邻; (N, P, 3) + (N, P, 3) => (N, P, K*D, 2) N=batch_num; P=point_num; 最后一维两个数:batch_indice + point_indice indices = indices_dilated[:, :, :: D, :] # (N, P, K, 2) 隔D取样,增加receptive field if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_pts = tf.gather_nd( pts, indices, name=tag + 'nn_pts' ) # (N, P, 3) + (N, P, K, 2) => (N, P, K, 3) indices维度比pts高,最后一维做索引,batch_indice->N, point_indice->P, 最后剩下(N, P, 3)里的3是坐标,补进去返回形状是 (N, P, K, 3) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) nn_pts_local = tf.subtract( nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3),减去中心点的坐标系,得到局部坐标系 # Prepare features to be transformed # 2个全连接层 对点的相对坐标系升维,(N, P, K, 3) => (N, P, K, C_pts_fts) # 升维的目的是为了和前面的特征拼接 (ct.) nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) # 如果第一次没有feature,就只使用从局部得到的点特征,如果有之前的特征,就拼接二者(ct.) if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') # 拼接坐标特征和之前的特征 if with_X_transformation: ######################## X-transformation ######################### # (N, P, K, 3) > (N, P, 1, K * K) ; kernel size 是(1, K) X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) # (N, P, 1, K * K) > (N, P, K, K) X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') # (N, P, K, K) > (N, P, 1, K * K) 可能为了较少参数量用depthwise_conv2d换conv2d X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) # (N, P, 1, K * K) > (N, P, K, K) X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') # (N, P, K, K) > (N, P, 1, K * K) X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) # (N, P, 1, K * K) > (N, P, K, K) 最后的x矩阵 (K*K) X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') # 用得到的 x 矩阵乘以之前的特征,所以这里其实是有2个分支,一个计算x矩阵,另外一个计算特征;(ct.) # (N, P, K, K) * (N, P, K, C_pts_fts) = (N, P, K, C_pts_fts) fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input # 最后的分离卷积 fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier ) # (N, P, K, C_pts_fts) -> (N, P, 1, C) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') # (N, P, C) # 用代表点全局位置信息, if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense( fts_global_0, C // 4, tag + 'fts_global', is_training ) # (N, P, C//4) 最后一层时作者使receptive field < 1让model更会捕捉local feature; 从“subvolume supervision”得到启发 to further address the over-fitting problem.; return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def __init__(self, points, features, num_class, is_training, setting, task): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.sampling == 'fps': from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] P = layer_param['P'] C = layer_param['C'] links = layer_param['links'] if setting.sampling != 'random' and links: print('Error: flexible links are supported only when random sampling is used!') exit() # get k-nearest points pts = self.layer_pts[-1] fts = self.layer_fts[-1] if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = self.layer_pts[-1] else: if setting.sampling == 'fps': indices = tf_sampling.farthest_point_sample(P, pts) qrs = tf_sampling.gather_point(pts, indices) # (N,P,3) elif setting.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) elif setting.sampling == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() self.layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method, layer_idx != len(xconv_params) - 1) fts_list = [] for link in links: fts_from_link = self.layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice_' + str(-link)) C_forward = math.ceil(fts_slice.get_shape().as_list()[-1] / (-link)) fts_forward = pf.dense(fts_slice, C_forward, tag + 'fts_forward_' + str(-link), is_training) fts_list.append(fts_forward) if fts_list: fts_list.append(fts_xconv) self.layer_fts.append(tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: self.layer_fts.append(fts_xconv) if task == 'segmentation': for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop) if task == 'classification': fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keep_dims=True, name='fc_mean') self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean) self.logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None) self.probs = tf.nn.softmax(self.logits, name='probs')
def __init__(self, points, features, images, image_xy, num_class, is_training, setting, task): print("points:", points) print("features:", features) xconv_params = setting.xconv_params print("xconv_params:", xconv_params) fc_params = setting.fc_params print("fc_params:", fc_params) with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] point_num = tf.shape(points)[1] if setting.with_fps: from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: C_fts = xconv_params[0][-1] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K, D, P, C = layer_param fts = self.layer_fts[-1] # get k centroid points pts = self.layer_pts[-1] if P == -1: qrs = self.layer_pts[-1] else: if setting.with_fps: fps_indices = tf_sampling.farthest_point_sample(P, pts) batch_indices = tf.tile( tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1)) fps_indices_g = tf.concat( [batch_indices, tf.expand_dims(fps_indices, -1)], axis=-1) qrs = tf.gather_nd(pts, fps_indices_g, name=tag + 'qrs') # (N, P, 3) else: qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) self.layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1][-1] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) self.layer_fts.append(fts_xconv) if task == 'segmentation': for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K, D, pts_layer_idx, qrs_layer_idx = layer_param pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[ pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] _, _, P, C = xconv_params[qrs_layer_idx] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): channel_num, drop_rate = layer_param fc = pf.dense(self.fc_layers[-1], channel_num, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, drop_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop) if task == 'classification': fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keep_dims=True, name='fc_mean') self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean) self.logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None) self.probs = tf.nn.softmax(self.logits, name='probs')
def deconv_new(pts, fts, qrs, tag, N, K, radius, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, D=1, sorting_method=None, with_global=False, knn=False): """ pts: points of previous layer, e.g.,(B, N/2, 3) fts: point features of previous layer, e.g.,(B, N/2, C) qrs: selected representative points of this layer, e.g.,(B, N, 3) N: batch_size, K: neighbor_size, D: dilation parameter, P: the number of selected representative points, C: output feature number per point, C_pts_fts: feature number for each local point radius: float32, the radius of query ball search knn: True: knn; False: query ball """ dist, idx = three_nn(qrs, pts) dist = tf.maximum(dist, 1e-10) norm = tf.reduce_sum((1.0 / dist), axis=2, keep_dims=True) norm = tf.tile(norm, [1, 1, 3]) weight = (1.0 / dist) / norm interpolated_fts = three_interpolate(fts, idx, weight) # (B, N, C) if knn: _, indices_dilated = pf.knn_indices_general(qrs, qrs, K * D, True) indices = indices_dilated[:, :, ::D, :] nn_pts = tf.gather_nd(qrs, indices, name=tag + 'nn_pts') # (B, N, K, 3) nn_fts_from_prev = tf.gather_nd(interpolated_fts, indices, name=tag + 'nn_fts') else: indices, pts_cnt = query_ball_point(radius, K, qrs, qrs) nn_pts = group_point(qrs, indices) nn_fts_from_prev = group_point(interpolated_fts, indices) nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (B, N, 1, 3) nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (B, N, K, 3) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_pts_local, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training) nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training) nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') if with_X_transformation: ######################## X-transformation ######################### X_0 = pf.conv2d(nn_pts_local, K * K, tag + 'X_0', is_training, (1, K)) # following paper X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K)) # following paper X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), activation=None) # following paper X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d
def xconv_cov(cov, pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method=None, with_global=False): """ cov: point covariance (B, N, 9) of previous layer pts: points of previous layer, fts: point features of previous layer, qrs: selected representative points of this layer, N: batch_size, K: neighbor_size, D: dilation parameter, P: the number of selected representative points, C: output feature number per point, C_pts_fts: feature number for each local point """ if D == 1: _, indices = pf.knn_indices_general(qrs, pts, K, True) else: _, indices_dilated = pf.knn_indices_general(qrs, pts, K * D, True) indices = indices_dilated[:, :, ::D, :] if sorting_method is not None: indices = pf.sort_points(pts, indices, sorting_method) nn_covs = tf.gather_nd(cov, indices, name=tag + 'nn_pts') # (N, P, K, 9) # nn_pts_center = tf.expand_dims(qrs, axis=2, name=tag + 'nn_pts_center') # (N, P, 1, 3) # nn_pts_local = tf.subtract(nn_pts, nn_pts_center, name=tag + 'nn_pts_local') # (N, P, K, 3) # Prepare features to be transformed nn_fts_from_pts_0 = pf.dense(nn_covs, C_pts_fts, tag + 'nn_fts_from_pts_0', is_training, with_bn=False) # following paper nn_fts_from_pts = pf.dense(nn_fts_from_pts_0, C_pts_fts, tag + 'nn_fts_from_pts', is_training, with_bn=False) # following paper if fts is None: nn_fts_input = nn_fts_from_pts else: nn_fts_from_prev = tf.gather_nd(fts, indices, name=tag + 'nn_fts_from_prev') nn_fts_input = tf.concat([nn_fts_from_pts, nn_fts_from_prev], axis=-1, name=tag + 'nn_fts_input') if with_X_transformation: ######################## X-transformation ######################### X_0 = pf.conv2d(nn_covs, K * K, tag + 'X_0', is_training, (1, K), with_bn=False) # following paper X_0_KK = tf.reshape(X_0, (N, P, K, K), name=tag + 'X_0_KK') X_1 = pf.depthwise_conv2d(X_0_KK, K, tag + 'X_1', is_training, (1, K), with_bn=False) # following paper X_1_KK = tf.reshape(X_1, (N, P, K, K), name=tag + 'X_1_KK') X_2 = pf.depthwise_conv2d(X_1_KK, K, tag + 'X_2', is_training, (1, K), with_bn=False, activation=None) # following paper X_2_KK = tf.reshape(X_2, (N, P, K, K), name=tag + 'X_2_KK') fts_X = tf.matmul(X_2_KK, nn_fts_input, name=tag + 'fts_X') ################################################################### else: fts_X = nn_fts_input fts_conv = pf.separable_conv2d(fts_X, C, tag + 'fts_conv', is_training, (1, K), depth_multiplier=depth_multiplier, with_bn=True) fts_conv_3d = tf.squeeze(fts_conv, axis=2, name=tag + 'fts_conv_3d') if with_global: fts_global_0 = pf.dense(qrs, C // 4, tag + 'fts_global_0', is_training, with_bn=True) fts_global = pf.dense(fts_global_0, C // 4, tag + 'fts_global_', is_training, with_bn=True) return tf.concat([fts_global, fts_conv_3d], axis=-1, name=tag + 'fts_conv_3d_with_global') else: return fts_conv_3d