def pointnet_sa_module(xyz, points, npoint, radius, nsample, mlp, mlp2, group_all, is_training, bn_decay, scope, bn=True, pooling='max', tnet_spec=None, knn=False, use_xyz=True): ''' PointNet Set Abstraction (SA) Module Input: xyz: (batch_size, ndataset, 3) TF tensor points: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling radius: float32 -- search radius in local region nsample: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point mlp2: list of int32 -- output size for MLP on each region group_all: bool -- group all points into one PC if set true, OVERRIDE npoint, radius and nsample settings use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor idx: (batch_size, npoint, nsample) int32 -- indices for local regions ''' with tf.variable_scope(scope) as sc: if group_all: nsample = xyz.get_shape()[1].value # (batch_size, 1, ndataset, 3+channel) new_xyz, new_points, idx, grouped_xyz = sample_and_group_all(xyz, points, use_xyz) else: # new_points:(batch_size, npoint, nsample, 3+channel) TF tensor new_xyz, new_points, idx, grouped_xyz = sample_and_group(npoint, radius, nsample, xyz, points, tnet_spec, knn, use_xyz) for i, num_out_channel in enumerate(mlp): #index and numerate # 一个mlp(多层感知机) new_points = tf_util.conv2d(new_points, num_out_channel, [1,1], padding='VALID', stride=[1,1], bn=bn, is_training=is_training, scope='conv%d'%(i), bn_decay=bn_decay) ### pooling 对采样维进行降维 if pooling=='avg': new_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1') elif pooling=='weighted_avg': with tf.variable_scope('weighted_avg1'): dists = tf.norm(grouped_xyz,axis=-1,ord=2,keep_dims=True) exp_dists = tf.exp(-dists * 5) weights = exp_dists/tf.reduce_sum(exp_dists,axis=2,keep_dims=True) # (batch_size, npoint, nsample, 1) new_points *= weights # (batch_size, npoint, nsample, mlp[-1]) new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True) elif pooling=='max': #参数2--reduction_indices:在哪一维上求解 如果keep_dims为true,则减小的维度将保留为长度1 new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True) # elif pooling=='min': new_points = tf_util.max_pool2d(-1*new_points, [1,nsample], stride=[1,1], padding='VALID', scope='minpool1') elif pooling=='max_and_avg': avg_points = tf_util.max_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='maxpool1') max_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1') new_points = tf.concat([avg_points, max_points], axis=-1) if mlp2 is None: mlp2 = [] for i, num_out_channel in enumerate(mlp2): new_points = tf_util.conv2d(new_points, num_out_channel, [1,1], padding='VALID', stride=[1,1], bn=bn, is_training=is_training, scope='conv_post_%d'%(i), bn_decay=bn_decay) new_points = tf.squeeze(new_points, [2]) # 压缩数目为1的维度 (batch_size, npoints, mlp2[-1]) return new_xyz, new_points, idx # mew_xyz:中心点 new_points:中心点采样后计算得到的featrue
def pointnet_sa_module(xyz, points, npoint, radius, nsample, mlp, mlp2, group_all, is_training, bn_decay, scope, bn=True, pooling='max', sort_k=None, tnet_spec=None, knn=False, use_xyz=True): ''' PointNet Set Abstraction (SA) Module Input: xyz: (batch_size, ndataset, 3) TF tensor points: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling radius: float32 -- search radius in local region nsample: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point mlp2: list of int32 -- output size for MLP on each region group_all: bool -- group all points into one PC if set true, OVERRIDE npoint, radius and nsample settings use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor idx: (batch_size, npoint, nsample) int32 -- indices for local regions ''' with tf.variable_scope(scope) as sc: if group_all: nsample = xyz.get_shape()[1].value new_xyz, new_points, idx, grouped_xyz = sample_and_group_all(xyz, points, use_xyz) else: new_xyz, new_points, idx, grouped_xyz = sample_and_group(npoint, radius, nsample, xyz, points, tnet_spec, knn, use_xyz) for i, num_out_channel in enumerate(mlp): new_points = tf_util.conv2d(new_points, num_out_channel, [1,1], padding='VALID', stride=[1,1], bn=bn, is_training=is_training, scope='conv%d'%(i), bn_decay=bn_decay) if pooling=='avg': new_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1') elif pooling=='weighted_avg': with tf.variable_scope('weighted_avg1'): dists = tf.norm(grouped_xyz,axis=-1,ord=2,keep_dims=True) exp_dists = tf.exp(-dists * 5) weights = exp_dists/tf.reduce_sum(exp_dists,axis=2,keep_dims=True) # (batch_size, npoint, nsample, 1) new_points *= weights # (batch_size, npoint, nsample, mlp[-1]) new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True) elif pooling=='max': new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True) elif pooling=='min': new_points = tf_util.max_pool2d(-1*new_points, [1,nsample], stride=[1,1], padding='VALID', scope='minpool1') elif pooling=='max_and_avg': max_points = tf.reduce_max(new_points, axis=[2], keep_dims=True, name='maxpool1') avg_points = tf.reduce_mean(new_points, axis=[2], keep_dims=True, name='avgpool1') new_points = tf.concat([avg_points, max_points], axis=-1) elif pooling=='sort': new_points = tf.transpose(tf.nn.top_k((tf.transpose(new_points, perm=[0, 1, 3, 2])), sort_k)[0], perm=[0, 1, 3, 2]) # (?, 2048, 1, 64) print new_points.get_shape() if mlp2 is None: mlp2 = [] for i, num_out_channel in enumerate(mlp2): new_points = tf_util.conv2d(new_points, num_out_channel, [1,1], padding='VALID', stride=[1,1], bn=bn, is_training=is_training, scope='conv_post_%d'%(i), bn_decay=bn_decay) if pooling != 'sort': new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1]) print new_points.get_shape() return new_xyz, new_points, idx
def feature_transform_net(inputs, is_training, bn_decay=None, K=64): """ Feature Transform Net, input is BxNx1xK Return: Transformation matrix of size KxK """ batch_size = inputs.get_shape()[0].value num_point = inputs.get_shape()[1].value net = tf_util.conv2d(inputs, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) #net = tf_util.conv2d(net, 128, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='tconv2', bn_decay=bn_decay) #net = tf_util.conv2d(net, 1024, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='tconv3', bn_decay=bn_decay) maxpool_net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='ttmaxpool') avg_pool2d_net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='ttavgpool') net = tf.concat([avg_pool2d_net, maxpool_net], axis=3) net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_feat') as sc: weights = tf.get_variable('weights', [256, K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, K, K]) return transform
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=6) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1,6], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) print("net = ", net.shape) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling max_net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') avg_net = tf_util.avg_pool2d(net, [num_point,1], padding='VALID', scope='avgpool') max_avg_net = tf.concat([max_net, avg_net], 3) net = tf.reshape(max_avg_net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 32, [1,9], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) #net = tf_util.conv2d(net, 128, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='conv2', bn_decay=bn_decay) #net = tf_util.conv2d(net, 128, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='conv3', bn_decay=bn_decay) #net = tf.concat([net,net1, net2],axis=-1) #net = tf_util.conv2d(net, 128, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='conv4', bn_decay=bn_decay) #net = tf_util.conv2d(net, 1024, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling maxpool_net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') avg_pool2d_net = tf_util.avg_pool2d(net, [num_point,1], padding='VALID', scope='avgpool') net = tf.concat([avg_pool2d_net, maxpool_net], axis=3) #net = tf.concat([tf.reduce_sum(net,axis=1,keep_dims=True), net1], axis=3) # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def pointnet_avg(source_point_cloud, template_point_cloud, is_training, bn_decay=None,out_features=1024): point_cloud = tf.concat([source_point_cloud, template_point_cloud], 0) batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, out_features, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) source_global_feature = tf.slice(net, [0, 0], [int(batch_size / 2), out_features]) template_global_feature = tf.slice(net, [int(batch_size / 2), 0], [int(batch_size / 2), out_features]) return source_global_feature, template_global_feature
def input_transform_net(point_cloud, is_training, bn_decay=None, K=9): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 128, [1, 9], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) #net = tf_util.conv2d(net, 128, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='tconv2', bn_decay=bn_decay) #net = tf_util.conv2d(net, 1024, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='tconv3', bn_decay=bn_decay) maxpool_net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') avg_pool2d_net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='tavgpool') net = tf.concat([avg_pool2d_net, maxpool_net], axis=3) net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: assert (K == 9) weights = tf.get_variable('weights', [256, 9 * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [9 * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 9, K]) return transform
def get_model(point_cloud, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) # CONV net = tf_util.conv2d(input_image, 64, [1, 12], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) points_feat1 = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # MAX and AVG pc_feat1_max = tf_util.max_pool2d(points_feat1, [num_point, 1], padding='VALID', scope='maxpool1') pc_feat1_avg = tf_util.avg_pool2d(points_feat1, [num_point, 1], padding='VALID', scope='avgpool1') pc_feat1 = tf.concat([pc_feat1_max, pc_feat1_avg], 3) # FC pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1]) pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) pc_feat1 = tf_util.fully_connected(pc_feat1, 128, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) print(pc_feat1) # CONCAT pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1]) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) # CONV net = tf_util.conv2d(points_feat1_concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6') net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7') net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 13, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv8') net = tf.squeeze(net, [2]) return net
def conv_network(point_cloud, is_training, scope, channels_out=1024, bn_decay=None): point_cloud = tf.expand_dims(point_cloud, 2) net, kernel = tf_util.inception(point_cloud, 64, scope=f'{scope}_1', kernel_heights=[1, 3, 5, 7], dilations=[3, 1], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 3, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training, return_kernel=True) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope=f'{scope}_1_max_pool', stride=[2, 1], padding='SAME') net = tf_util.inception(net, 128, scope=f'{scope}_2', kernel_heights=[1, 3, 5, 7], dilations=[3, 1], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 3, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope=f'{scope}_2_max_pool', stride=[2, 1], padding='SAME') net = tf_util.inception(net, 128, scope=f'{scope}_3', kernel_heights=[1, 3, 5, 7], dilations=[3, 1], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 3, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope=f'{scope}_3_max_pool', stride=[2, 1], padding='SAME') net = tf_util.inception(net, 256, scope=f'{scope}_4', kernel_heights=[1, 3, 5, 7], dilations=[3, 1], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 3, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope=f'{scope}_4_max_pool', stride=[2, 1], padding='SAME') net = tf_util.inception(net, 512, scope=f'{scope}_5', kernel_heights=[1, 3, 5, 7], dilations=[3, 1], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 3, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope=f'{scope}_5_max_pool', stride=[2, 1], padding='SAME') net = tf_util.inception(net, channels_out, scope=f'{scope}_6', kernel_heights=[1, 3, 5, 7], dilations=[3, 1], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 3, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.avg_pool2d(net, kernel_size=[net.shape[1], 1], scope=f'{scope}_GAP', stride=[net.shape[1], 1], padding='SAME') net = tf.squeeze(net) return net, kernel
def get_instance_seg_v1_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v1 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value net = tf.expand_dims(point_cloud, 2) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) #global_feat = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') global_feat = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='avgpool') global_feat = tf.concat( [global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3) global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand]) net = tf_util.conv2d(concat_feat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5) logits = tf_util.conv2d(net, 2, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv10') logits = tf.squeeze(logits, [2]) # BxNx2 return logits, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \ batch_size, num_point, weight_decay, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} with tf.variable_scope('transform_net1') as sc: K = 6 transform = get_transform(point_cloud, is_training, bn_decay, K) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) out1 = tf_util.conv2d(input_image, 64, [1,6], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out3 = tf_util.conv2d(out2, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128]) net_transformed = tf.matmul(squeezed_out3, transform) net_transformed = tf.expand_dims(net_transformed, [2]) out4 = tf_util.conv2d(net_transformed, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point,1], padding='VALID', scope='maxpool') out_avg = tf_util.avg_pool2d(out5, [num_point,1], padding='VALID', scope='avgpool') out_max = tf.concat(axis=3, values=[out_max, out_avg]) # classification network net = tf.reshape(out_max, [batch_size, -1]) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1') net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3') # segmentation network one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5]) net2 = tf_util.conv2d(concat, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay) net2 = tf_util.conv2d(net2, part_num, [1,1], padding='VALID', stride=[1,1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net, net2, end_points
def vgg16(input, is_training, output_dim, bn_decay=None): """ objective: build the network param: input: image patch in brg (after pre-processing), [batch, height, width, 3], is_training: training or testing bn_decay: None output_dim: final feature dimension """ # block 1 (128x128x3 -> 64x64x64) conv1_1 = tf_util.conv2d(input, 64, [3, 3], scope='conv1_1', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) conv1_2 = tf_util.conv2d(conv1_1, 64, [3, 3], scope='conv1_2', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) pool1 = tf_util.max_pool2d(conv1_2, [2, 2], scope='pool1', stride=[2, 2], padding='VALID') # block 2 (64x64x64 -> 32x32x128) conv2_1 = tf_util.conv2d(pool1, 128, [3, 3], scope='conv2_1', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) conv2_2 = tf_util.conv2d(conv2_1, 128, [3, 3], scope='conv2_2', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) pool2 = tf_util.max_pool2d(conv2_2, [2, 2], scope='pool2', stride=[2, 2], padding='VALID') # block 3 (32x32x128 -> 16x16x256) conv3_1 = tf_util.conv2d(pool2, 256, [3, 3], scope='conv3_1', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) conv3_2 = tf_util.conv2d(conv3_1, 256, [3, 3], scope='conv3_2', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) conv3_3 = tf_util.conv2d(conv3_2, 256, [3, 3], scope='conv3_3', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) pool3 = tf_util.max_pool2d(conv3_3, [2, 2], scope='pool3', stride=[2, 2], padding='VALID') # block 4 (16x16x256 -> 16x16x512 -> 1x1x512) conv4_1 = tf_util.conv2d(pool3, 512, [3, 3], scope='conv4_1', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) conv4_2 = tf_util.conv2d(conv4_1, 512, [3, 3], scope='conv4_2', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) conv4_3 = tf_util.conv2d(conv4_2, 512, [3, 3], scope='conv4_3', stride=[1, 1], padding='SAME', is_training=is_training, bn_decay=bn_decay) # net = tf_util.max_pool2d(net, [2,2], scope='pool4',stride=[2,2],padding='VALID') pool4 = tf_util.avg_pool2d(conv4_3, [16, 16], scope='pool4', stride=[16, 16], padding='VALID') shape = pool4.get_shape().as_list() dim = 1 for d in shape[1:]: dim *= d pool4 = tf.reshape(pool4, [-1, dim]) print(pool4.shape) # block 5 (16x16x512 -> 1x512 -> 1x128) fc1 = tf_util.fully_connected(pool4, 512, scope='fc1', is_training=is_training) # fc1 = tf_util.dropout(fc1, keep_prob=0.7, is_training=is_training, # scope='dp1') #fc2 = tf_util.fully_connected(fc1, output_dim, scope='fc2', is_training = is_training) fc2 = tf_util.fully_connected(fc1, output_dim, activation_fn=None, scope='fc2', is_training=is_training) out = tf.nn.l2_normalize(fc2, dim=1) return out
def get_model(point_cloud, is_training, n_classes, bn_decay=None, weight_decay=None, inception=True, **kwargs): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value channels_size = point_cloud.get_shape()[2].value end_points = {} if point_cloud.shape[2] > 3: l0_xyz = point_cloud[:, :, :3] l0_points = point_cloud[:, :, 3:] else: l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz input_image = tf.expand_dims(point_cloud, 2) net, conv_ker = tf_util.inception(input_image, 64, scope='seq_conv1', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 2, 2, 2], return_kernel=True, bn=True, bn_decay=bn_decay, is_training=is_training) end_points['conv_ker'] = conv_ker net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_1', stride=[2,1], padding='SAME') net = tf_util.inception(net, 64, scope='seq_conv2', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[3, 2, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_2', stride=[2,1], padding='SAME') net = tf_util.inception(net, 128, scope='seq_conv3', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[3, 2, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_3', stride=[2,1], padding='SAME') net = tf_util.inception(net, 256, scope='seq_conv4', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[3, 2, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_4', stride=[2,1], padding='SAME') net = tf_util.inception(net, 512, scope='seq_conv5', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[3, 2, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.max_pool2d(net, kernel_size=[2, 1], scope='seq_maxpool_5', stride=[2,1], padding='SAME') net = tf_util.inception(net, 1024, scope='seq_conv6', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[3, 2, 2, 1], bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.avg_pool2d(net, kernel_size=[net.shape[1], 1], scope='seq_global_avg_pool', stride=[net.shape[1],1], padding='SAME') # # Symmetric function: max pooling # net = tf_util.max_pool2d(net, [num_point,1], # padding='VALID', scope='maxpool') conv_net = tf.squeeze(net) # Set abstraction layers # Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4). # So we only use NCHW for layer 1 until this issue can be resolved. l1_xyz, l1_points, l1_indices, pt_ker = pointnet_sa_module(l0_xyz, l0_points, npoint=256, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) end_points['pt_ker'] = pt_ker l2_xyz, l2_points, l2_indices, _ = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=128, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') net = tf.reshape(l3_points, [batch_size, -1]) net = tf.concat([conv_net, net], axis=-1) end_points['feature_vector'] = net # Fully connected layers net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1') # my change net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, weight_decay=weight_decay) net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay, weight_decay=weight_decay) net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, n_classes, activation_fn=None, scope='fc3') return net, end_points
def ec_to_weights(ec, mask, neighbornum, C=3, bn_decay=None, is_training=None): # ec B N K*C batch_size = ec.get_shape()[0] point_num = ec.get_shape()[1] #K = ec.get_shape()[2] #ec_feature = edge_unit_with_ec(ec,mask,'max', config.neighbor_num, 32, scope='ec_conv1', bn=True, is_training=is_training,bn_decay=bn_decay) #ec_feature = tf.expand_dims(ec_feature, -1) #ec = tf.reshape(ec,[batch_size,point_num,neighbornum, C]) # B N K C with tf.variable_scope('edge_net_without_pooling') as sc: net, kernel = edge_unit(ec, mask, 'max', config.neighbor_num, 32, scope='conv1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.conv2d(ec, 64, [1, C], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='ec_conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='ec_conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='ec_conv3', bn_decay=bn_decay) net = tf_util.avg_pool2d(net, [1, 1024], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: if (K == 4): weights = tf.get_variable('weights', [256, N * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [4 * 4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 4, 4]) return transform
def get_model(point_cloud, is_training, num_class, weight_decay=None, bn_decay=None, scname=''): ''' input: BxNxF output:BxNx(cats*segms) ''' batch_size = point_cloud.get_shape()[0] num_point = point_cloud.get_shape()[1] num_feat = point_cloud.get_shape()[2] k = 10 adj = tf_util.pairwise_distanceR(point_cloud[:, :, :3]) n_heads = 1 nn_idx = tf_util.knn(adj, k=k) net, locals_transform, coefs = gap_block(k, n_heads, nn_idx, point_cloud, point_cloud, ('filter0', 16), bn_decay, weight_decay, is_training, scname) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet01' + scname, bn_decay=bn_decay) net01 = net net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet02' + scname, bn_decay=bn_decay) net02 = net adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=k) adj_conv = nn_idx n_heads = 1 net, locals_transform1, coefs2 = gap_block(k, n_heads, nn_idx, net, point_cloud, ('filter1', 128), bn_decay, weight_decay, is_training, scname) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet11' + scname, bn_decay=bn_decay) net11 = net net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet12' + scname, bn_decay=bn_decay) net12 = net net = tf.concat( [net01, net02, net11, net12, locals_transform, locals_transform1], axis=-1) net = tf_util.conv2d(net, 3, [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='agg' + scname, bn_decay=bn_decay) net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='avgpool' + scname) max_pool = net net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, activation_fn=tf.nn.relu, scope='fc1' + scname, bn_decay=bn_decay) net = tf_util.fully_connected(net, 128, bn=True, is_training=is_training, activation_fn=tf.nn.relu, scope='fc2' + scname, bn_decay=bn_decay) net = tf_util.fully_connected(net, num_class, activation_fn=None, scope='fc3' + scname) net = tf.squeeze(net) return net, max_pool
def get_model(point_cloud, query_points, is_training, bn_decay=None): """ range regression PointNet, input is BxNx3(point_cloud) and Bx2(query_points), output Bx1 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} # query_points = tf.tile( tf.expand_dims(query_points, [1]), [1, num_point, 1]) # Now, query_points is with shape BxNx2 query_points = tf.concat( [query_points, tf.zeros([batch_size, num_point, 1])], 2) # Now, query_points is with shape BxNx3 # point_cloud_ab = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 2]) # alpha and beta, shape:BxNx2 # point_cloud_range = tf.slice(point_cloud, [0, 0, 2], [-1, -1, 1]) # range, shape:BxNx1 # shift_ab = point_cloud_ab - query_points shifted_pl = point_cloud - query_points # with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(shifted_pl, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(shifted_pl, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: average pooling net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='avgpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=False, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=False, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 1, activation_fn=None, scope='fc3') return net, end_points
def get_model_fine_tuning_test(point_cloud, point_coords_in_voxels, num_scale, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) points_feat1 = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # PYRAMID START # # m x n x 1024 points_feat1 = tf.squeeze(points_feat1, [2]) print(points_feat1) # m x n x (4 x 128 = 512) points_feat1_concat = pyramid_nets.pyramid_convert_layer( points_feat1, point_coords_in_voxels, num_scale, [256], bn=True, is_training=is_training, bn_decay=bn_decay) print(points_feat1_concat) # m x n x 1 x 512 points_feat1_concat = tf.expand_dims(points_feat1_concat, [2]) # Concat pyramid global and local features points_feat1 = tf.expand_dims(points_feat1, [2]) point_feat_concat = tf.concat(axis=3, values=[points_feat1, points_feat1_concat]) # PYRAMID END # # Symmetric function: max pooling #net = tf_util.max_pool2d(point_feat_concat, [num_point,1], padding='VALID', scope='pyramid_maxpool') net = tf_util.avg_pool2d(point_feat_concat, [num_point, 1], padding='VALID', scope='pyramid_maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='pyramid_fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='pyramid_dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='pyramid_fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='pyramid_dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='pyramid_fc3') return net, end_points
def get_model( point_cloud, is_training, one_hot_labels, bn_decay=None, ): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tfgan.features.condition_tensor_from_onehot(net, one_hot_labels) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) #net = tfgan.features.condition_tensor_from_onehot(net, one_hot_labels) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, num_class, global_pl, params, weight_decay=None, bn_decay=None, scname=''): ''' input: BxNxF Use https://arxiv.org/pdf/1902.08570 as baseline output:BxNx(cats*segms) ''' batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -2) k = params[0] #adj = tf_util.pairwise_distance(point_cloud[:,:,:3]) adj = tf_util.pairwise_distanceR(point_cloud[:, :, :3]) n_heads = params[1] nn_idx = tf_util.knn(adj, k=k) net, locals_transform, coefs = gap_block(k, n_heads, nn_idx, point_cloud, point_cloud, ('filter0', params[2]), bn_decay, weight_decay, is_training, scname) net = tf_util.conv2d(net, params[3], [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='gapnet00', bn_decay=bn_decay) net00 = net net = tf_util.conv2d(net, params[4], [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet01' + scname, bn_decay=bn_decay) net01 = net net = tf_util.conv2d(net, params[5], [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet02' + scname, bn_decay=bn_decay) net02 = net adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=k) adj_conv = nn_idx n_heads = params[6] net, locals_transform1, coefs2 = gap_block(k, n_heads, nn_idx, net, point_cloud, ('filter1', params[7]), bn_decay, weight_decay, is_training, scname) net = tf_util.conv2d(net, params[8], [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='gapnet10', bn_decay=bn_decay) net10 = net net = tf_util.conv2d(net, params[9], [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet11' + scname, bn_decay=bn_decay) net11 = net net = tf_util.conv2d(net, params[10], [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='gapnet12' + scname, bn_decay=bn_decay) net12 = net global_expand = tf.reshape(global_pl, [batch_size, 1, 1, -1]) global_expand = tf.tile(global_expand, [1, num_point, 1, 1]) global_expand = tf_util.conv2d(global_expand, 16, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='global_expand' + scname, bn_decay=bn_decay) net = tf.concat([ net00, net01, net02, net11, net12, global_expand, locals_transform, locals_transform1 ], axis=-1) net = tf_util.conv2d(net, params[8], [1, 1], padding='VALID', stride=[1, 1], activation_fn=tf.nn.relu, bn=True, is_training=is_training, scope='agg' + scname, bn_decay=bn_decay) #net_tot = net net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='avgpool' + scname) expand = tf.tile(net, [1, num_point, 1, 1]) # net = tf.concat(axis=3, values=[expand, # net_tot, # ]) net = tf_util.conv2d(expand, params[11], [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay, is_dist=True) net = tf_util.dropout(net, keep_prob=0.6, is_training=is_training, scope='seg/dp1') net = tf_util.conv2d(net, params[11], [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay, is_dist=True) net = tf_util.dropout(net, keep_prob=0.6, is_training=is_training, scope='seg/dp2') net = tf_util.conv2d(net, params[12], [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv4', weight_decay=weight_decay, is_dist=True) net = tf_util.conv2d(net, num_class, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv5', weight_decay=weight_decay, is_dist=True) net = tf.cond(is_training, lambda: net, lambda: tf.nn.softmax(net)) net = tf.reshape(net, [batch_size, num_point, num_class]) return net, coefs, coefs2, adj_conv
def get_model_dss(point_cloud, is_training, is_outer, bn_decay=None, pool='max'): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} # with tf.variable_scope('transform_net1') as sc: # transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) # point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', activation_fn=tf.nn.relu, bn_decay=bn_decay) side_1 = tf_util.conv2d(tf.reduce_max(net, axis=3, keepdims=True), 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='side_conv2', activation_fn=None, bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', activation_fn=None, bn_decay=bn_decay) net = tf.nn.relu(net + side_1) side_2 = tf_util.conv2d(tf.reduce_max(net, axis=3, keepdims=True), 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='side_conv3', activation_fn=None, bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', activation_fn=None, bn_decay=bn_decay) net = tf.nn.relu(net + side_2) tf.reduce_max(net, axis=3, keepdims=True) # print(net.shape) # with tf.variable_scope('transform_net2') as sc: # transform = feature_transform_net(net, is_training, bn_decay, K=64) # end_points['transform'] = transform # net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) # net_transformed = tf.expand_dims(net_transformed, [2]) # net = tf_util.conv2d(net_transformed, 64, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='conv3', bn_decay=bn_decay) # net = tf_util.conv2d(net, 128, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='conv4', activation_fn=tf.nn.relu, bn_decay=bn_decay) # net = tf_util.conv2d(net, 1024, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='conv5', activation_fn=None, bn_decay=bn_decay) # Symmetric function: max pooling if pool == 'max': net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') elif pool == 'median': net = tf_util.median_pool2d(net) elif pool == 'median_inner': net = tf.cond( is_outer, lambda: tf_util.median_pool2d(net), lambda: tf_util. sum_pool2d(net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf.stop_gradient( tf_util.median_pool2d(net) - tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_2'))) elif pool == 'median_outer': net = tf.cond( is_outer, lambda: tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf. stop_gradient( tf_util.median_pool2d(net) - tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_2')), lambda: tf_util.median_pool2d(net)) elif pool == 'median_both': net = tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf.stop_gradient( tf_util.median_pool2d(net) - tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_2')) elif pool == 'avg': net = tf_util.avg_pool2d(net, [num_point, 1], padding='VALID', scope='avgpool') elif pool == 'inner': net = tf.cond( is_outer, lambda: tf_util.max_pool2d( net, [num_point, 1], padding='VALID', scope='maxpool'), lambda: tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf. stop_gradient( tf_util.max_pool2d( net, [num_point, 1], padding='VALID', scope='maxpool') - tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_2'))) elif pool == 'outer': net = tf.cond( is_outer, lambda: tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf. stop_gradient( tf_util.max_pool2d( net, [num_point, 1], padding='VALID', scope='maxpool') - tf_util.sum_pool2d(net, [num_point, 1], padding='VALID', scope='sumpool_2')), lambda: tf_util. max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool')) elif pool == 'both': net = tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_1') + tf.stop_gradient( tf_util.max_pool2d( net, [num_point, 1], padding='VALID', scope='maxpool') - tf_util.sum_pool2d( net, [num_point, 1], padding='VALID', scope='sumpool_2')) elif pool == 'mil': net = tf_util.mil_pooling(net, scope='milpool', is_training=is_training) elif pool == 'gate': net = tf_util.gate_pooling(net, scope='gatepool', is_training=is_training) elif pool == 'pma': net = tf_util.pma_pooling(net, scope='pmapool', is_training=is_training) elif pool == 'deepsym': net = tf_util.deep_sym(net, scope='deepsym', is_training=is_training) elif pool == 'fspool': net = tf_util.fspool(net, scope='fspool') elif pool == 'gate_v2': net = tf_util.gate_pooling_v2(net, scope='gatepool', is_training=is_training) # print(net.shape) end_points['global'] = net # print(net.shape) net = tf.reshape(net, [batch_size, -1]) # net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, # scope='fc1', bn_decay=bn_decay) # net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, activation_fn=tf.nn.relu, scope='fc2', bn_decay=bn_decay) # net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def pointnet_sa_module_spec(xyz, points, npoint, radius, nsample, mlp, mlp2, group_all, is_training, bn_decay, scope, bn=True, pooling='max', tnet_spec=None, knn=False, use_xyz=True, spec_conv_type='mlp', structure='spec', useloc_covmat=True, csize=None): ''' PointNet Set Abstraction (SA) Module Input: xyz: (batch_size, ndataset, 3) TF tensor points: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling radius: float32 -- search radius in local region nsample: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point mlp2: list of int32 -- output size for MLP on each region group_all: bool -- group all points into one PC if set true, OVERRIDE npoint, radius and nsample settings use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features # special to spectrual conv useloc_covmat: use location xyz ONLY for local neighbourhood's covariance matrix computation; Set false to use all point features including xyz + network feats. Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor idx: (batch_size, npoint, nsample) int32 -- indices for local regions ''' with tf.variable_scope(scope) as sc: if group_all: nsample = xyz.get_shape()[1].value new_xyz, new_points, idx, grouped_xyz = sample_and_group_all( xyz, points, use_xyz) else: new_xyz, new_points, idx, grouped_xyz = sample_and_group( npoint, radius, nsample, xyz, points, tnet_spec, knn, use_xyz) # grouped_xyz this is local xyz wrt to each center point # dim B N K 3 # controls using what feature for covariance computation if useloc_covmat: local_cord = grouped_xyz else: local_cord = None if structure == 'spec': mlp_spec = mlp new_points, UT = spec_conv2d(inputs=new_points, num_output_channels=mlp_spec, local_cord=local_cord, bn=bn, is_training=is_training, scope='spec_conv%d' % (0), bn_decay=bn_decay) if structure == 'spec-modul': mlp_spec = mlp new_points, UT = spec_conv2d_modul(inputs=new_points, num_output_channels=mlp_spec, local_cord=local_cord, bn=bn, is_training=is_training, scope='spec_conv%d' % (0), bn_decay=bn_decay) if pooling == 'avg': new_points = tf_util.avg_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='avgpool1') elif pooling == 'weighted_avg': with tf.variable_scope('weighted_avg1'): dists = tf.norm(grouped_xyz, axis=-1, ord=2, keep_dims=True) exp_dists = tf.exp(-dists * 5) weights = exp_dists / tf.reduce_sum( exp_dists, axis=2, keep_dims=True) # (batch_size, npoint, nsample, 1) new_points *= weights # (batch_size, npoint, nsample, mlp[-1]) new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True) elif pooling == 'max': new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True) elif pooling == 'min': new_points = tf_util.max_pool2d(-1 * new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='minpool1') elif pooling == 'max_and_avg': avg_points = tf_util.max_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='maxpool1') max_points = tf_util.avg_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='avgpool1') new_points = tf.concat([avg_points, max_points], axis=-1) elif pooling == 'hier_cluster_pool': new_points = spec_hier_cluster_pool(new_points, pool_method='max', csize=csize) if mlp2 is None: mlp2 = [] for i, num_out_channel in enumerate(mlp2): new_points = tf_util.conv2d(new_points, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv_post_%d' % (i), bn_decay=bn_decay) new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1]) return new_xyz, new_points, idx
def pointnet_sa_module(xyz, points, xyz_feature, npoint, radius, nsample, mlp, mlp2, mlp3, group_all, is_training, bn_decay, scope, bn=True, pooling='max', tnet_spec=None, knn=False, use_xyz=False, end=False, use_xyz_feature=False): ''' PointNet Set Abstraction (SA) Module Input: xyz: (batch_size, ndataset, 3) TF tensor points: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling radius: float32 -- search radius in local region nsample: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point mlp2: list of int32 -- output size for MLP on each region group_all: bool -- group all points into one PC if set true, OVERRIDE npoint, radius and nsample settings use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor idx: (batch_size, npoint, nsample) int32 -- indices for local regions ''' with tf.variable_scope(scope) as sc: batch_size = xyz.get_shape()[0].value if group_all: nsample = xyz.get_shape()[1].value new_xyz, new_points, idx, grouped_xyz = sample_and_group_all( xyz, points, use_xyz) idx = tf.cast(idx, tf.int32) ########### additional xyz features if use_xyz_feature: xyz_feature = group_point(xyz_feature, idx) e, grouped_xyz = PCA_decompose(grouped_xyz) e = tf.tile(tf.expand_dims(e, 2), [1, 1, nsample, 1]) edge_feature = tf.concat( [relative_pos_encoding(tf.abs(grouped_xyz)), e], axis=-1) edge_feature = tf_util.conv2d(edge_feature, mlp3[0], [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='xyz_feature_%d' % (0), bn_decay=bn_decay) edge_feature = tf_util.conv2d(edge_feature, mlp3[1], [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='xyz_feature_%d' % (1), bn_decay=bn_decay) output_feature = tf.concat([xyz_feature, edge_feature], axis=-1) if end == False: xyz_feature = tf_util.conv2d(output_feature, mlp3[-1], [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='xyz_feature2', bn_decay=bn_decay) # we can try sum and mean xyz_feature = tf.reduce_max(xyz_feature, axis=[2], keep_dims=True, name='maxpool') xyz_feature = tf.squeeze(xyz_feature, [2]) new_points = tf.concat([new_points, output_feature], axis=-1) for i, num_out_channel in enumerate(mlp): new_points = tf_util.conv2d(new_points, num_out_channel, [1, 1], padding='SAME', stride=[1, 1], bn=bn, is_training=is_training, scope='conv%d' % (i), bn_decay=bn_decay) if pooling == 'avg': new_points = tf_util.avg_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='avgpool1') elif pooling == 'weighted_avg': with tf.variable_scope('weighted_avg1'): dists = tf.norm(grouped_xyz, axis=-1, ord=2, keep_dims=True) exp_dists = tf.exp(-dists * 5) weights = exp_dists / tf.reduce_sum( exp_dists, axis=2, keep_dims=True) # (batch_size, npoint, nsample, 1) new_points *= weights # (batch_size, npoint, nsample, mlp[-1]) new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True) elif pooling == 'max': new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True) elif pooling == 'min': new_points = tf_util.max_pool2d(-1 * new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='minpool1') elif pooling == 'max_and_avg': avg_points = tf_util.max_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='maxpool1') max_points = tf_util.avg_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='avgpool1') new_points = tf.concat([avg_points, max_points], axis=-1) if mlp2 is None: mlp2 = [] for i, num_out_channel in enumerate(mlp2): new_points = tf_util.conv2d(new_points, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv_post_%d' % (i), bn_decay=bn_decay) new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1]) return new_xyz, new_points, idx, xyz_feature