def create_decoder(self, features): """fully connected layers for classification with dropout""" with tf.variable_scope('decoder_cls', reuse=tf.AUTO_REUSE): features = fully_connected(features, 512, bn=True, scope='fc1', is_training=self.is_training) features = dropout(features, keep_prob=0.7, scope='dp1', is_training=self.is_training) features = fully_connected(features, 256, bn=True, scope='fc2', is_training=self.is_training) features = dropout(features, keep_prob=0.7, scope='dp2', is_training=self.is_training) pred = fully_connected(features, NUM_CLASSES, activation_fn=None, scope='fc3', is_training=self.is_training) return pred
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNetwork :param point_cloud: point cloud BxNx3 :param is_training: training flag :param bn_decay: decay flag :return: output model Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MPL implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symetric function: Max Pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3]) # Set Abstraction layers l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Layer 1 l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def create_decoder(self): net = conv2d(self.cat_feats, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv6') net = conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv7') net = dropout(net, keep_prob=0.7, is_training=self.is_training, scope='dp1') net = conv2d(net, 13, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv8') net = tf.squeeze(net, [2]) return net
def create_decoder(self, features): """fully connected layers for classification with dropout""" with tf.variable_scope('decoder_cls', reuse=tf.AUTO_REUSE): # self.linear1 = nn.Linear(args.emb_dims*2, 512, bias=False) features = tf_util.fully_connected(features, 512, bn=True, bias=False, activation_fn=tf.nn.leaky_relu, scope='linear1', is_training=self.is_training) features = tf_util.dropout(features, keep_prob=0.5, scope='dp1', is_training=self.is_training) # self.linear2 = nn.Linear(512, 256) features = tf_util.fully_connected(features, 256, bn=True, bias=True, activation_fn=tf.nn.leaky_relu, scope='linear2', is_training=self.is_training) features = tf_util.dropout(features, keep_prob=0.5, scope='dp2', is_training=self.is_training) # self.linear3 = nn.Linear(256, output_channels) pred = tf_util.fully_connected(features, NUM_CLASSES, bn=False, bias=True, activation_fn=None, scope='linear3', is_training=self.is_training) return pred
def get_model(point_cloud, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) # CONV net = tf_util.conv2d(input_image, 64, [1,9], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) points_feat1 = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # MAX pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point,1], padding='VALID', scope='maxpool1') # FC pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1]) pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) pc_feat1 = tf_util.fully_connected(pc_feat1, 128, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) print(pc_feat1) # CONCAT pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1]) print(points_feat1) print(pc_feat1_expand) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) print(points_feat1_concat) # CONV net = tf_util.conv2d(points_feat1_concat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6') net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7') net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 13, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv8') net = tf.squeeze(net, [2]) return net
def get_model(point_cloud, cls_label, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = tf.shape(point_cloud)[0] num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer2') cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1]) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points], axis=-1), l1_points, [128, 128], is_training, bn_decay, scope='fp_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def inference_multiview_all(images, n_classes=40, is_training=False): """ :param images: (N, V, W, H, C) :param is_training: is_training :param bn_decay: bn_decay :return: fc logits """ # batch_size = images.get_shape().as_list()[0] # n_views = images.get_shape().as_list()[1] # weight = images.get_shape().as_list()[2] # height = images.get_shape().as_list()[3] # dims = images.get_shape().as_list()[4] batch_size = images.shape[0].value n_views = images.shape[1].value weight = images.shape[2].value height = images.shape[3].value dims = images.shape[4].value # Get images (N*V, W, H, C) images = tf.reshape(images, [batch_size * n_views, weight, height, dims]) reuse = False conv1 = _conv('conv1', images, [11, 11, 3, 96], [1, 4, 4, 1], 'VALID', reuse=reuse) lrn1 = None pool1 = _maxpool('pool1', conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID') conv2 = _conv('conv2', pool1, [5, 5, 96, 256], group=2, reuse=reuse) lrn2 = None pool2 = _maxpool('pool2', conv2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID') conv3 = _conv('conv3', pool2, [3, 3, 256, 384], reuse=reuse) conv4 = _conv('conv4', conv3, [3, 3, 384, 384], group=2, reuse=reuse) conv5 = _conv('conv5', conv4, [3, 3, 384, 256], group=2, reuse=reuse) pool5 = _maxpool('pool5', conv5, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID') hwc = np.prod(pool5.get_shape().as_list()[1:]) flatten = tf.reshape(pool5, [batch_size * n_views, hwc]) view_pooling = tf.reshape(flatten, [batch_size, n_views, hwc]) view_pooling = tf.reduce_max(view_pooling, axis=1) # print 'pool5_vp', view_pooling.get_shape().as_list() fc6_b = _fc('fc6', view_pooling, 4096) fc6 = tf_util.dropout(fc6_b, is_training, scope='dp6', keep_prob=0.5) fc7 = _fc('fc7', fc6, 4096) fc7 = tf_util.dropout(fc7, is_training, scope='dp7', keep_prob=0.5) fc8 = _fc('fc8', fc7, n_classes) return fc8
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Set abstraction layers # Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4). # So we only use NCHW for layer 1 until this issue can be resolved. l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=32, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.layers.flatten(l3_points) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, num_class, hyperparams, bn_decay=None): """ Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} if hyperparams['use_color'] or hyperparams['use_z_feature']: feature_size = 3 * int(hyperparams['use_color']) + int( hyperparams['use_z_feature']) l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, feature_size]) else: l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Layer 1 l1_xyz, l1_points, l1_indices = pointnet_sa_module( l0_xyz, l0_points, npoint=hyperparams['l1_npoint'], radius=hyperparams['l1_radius'], nsample=hyperparams['l1_nsample'], mlp=[32, 32, 64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module( l1_xyz, l1_points, npoint=hyperparams['l2_npoint'], radius=hyperparams['l2_radius'], nsample=hyperparams['l2_nsample'], mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module( l2_xyz, l2_points, npoint=hyperparams['l3_npoint'], radius=hyperparams['l3_radius'], nsample=hyperparams['l3_nsample'], mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l4_xyz, l4_points, l4_indices = pointnet_sa_module( l3_xyz, l3_points, npoint=hyperparams['l4_npoint'], radius=hyperparams['l4_radius'], nsample=hyperparams['l4_nsample'], mlp=[256, 256, 512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model_other(point_cloud, is_training, bn_decay=None): """ Classification DGCNN, input is BxNxC, output BxCls B batch size N number of points per pointcloud C input channel: eg. x,y,z,SF,distance, minSF... Cls output class number """ batch_size = point_cloud.get_shape()[0] # .value end_points = {} # get MinSF index minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1)) # # 1. graph for transform net with only x,y,z adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:4]) # B N C=3 => B N N nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(point_cloud[:, :, 1:4], nn_idx=nn_idx, k=para.k) with tf.compat.v1.variable_scope('transform_net1') as sc: transform = input_transform_net_dgcnn(edge_feature, is_training, bn_decay, K=3) point_cloud_transform = tf.matmul(point_cloud[:, :, 1:4], transform) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) # B N 1 end_points['knn1'] = allSF_dist point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1) # # 2. graph for first EdgeConv with transform(x,y,z), SF, distance, minSF point_cloud_all = tf.concat(axis=2, values=[point_cloud_SF, point_cloud_transform, point_cloud[:, :, 4:para.dim]]) adj_matrix = tf_util.pairwise_distance(point_cloud_all) # B N C=6 nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(point_cloud_all, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn1', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net1 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn2'] = allSF_dist # # 3. graph for second EdgeConv with C = 64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn2', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net2 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn3'] = allSF_dist # # 4. graph for third EdgeConv with C = 64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn3', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net3 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn4'] = allSF_dist # # 5. graph for fourth EdgeConv with C = 64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn4', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net4 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn5'] = allSF_dist # # 6. MLP for all concatenate features 64+64+64+128 net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='agg', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=1, keepdims=True) # maxpooling B N C=1024 => B 1 1024 # # 7. MLP on global point cloud vector B 1 1024 net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, batch_size, num_point, weight_decay, bn_decay=None): """ Get ConvNet baseline model. :param point_cloud: :param input_label: :param is_training: :param cat_num: :param part_num: :param batch_size: :param num_point: :param weight_decay: :param bn_decay: :return: """ end_points = {} with tf.variable_scope('transform_net1') as sc: K = 3 transform = get_transform(point_cloud, is_training, bn_decay, k=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) out1 = tf_util.conv2d(input_image, 64, [1, K], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out3 = tf_util.conv2d(out2, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128]) net_transformed = tf.matmul(squeezed_out3, transform) net_transformed = tf.expand_dims(net_transformed, [2]) out4 = tf_util.conv2d(net_transformed, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point, 1], padding='VALID', scope='maxpool') # classification network net = tf.reshape(out_max, [batch_size, -1]) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1') net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3') # segmentation network one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5]) net2 = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay) net2 = tf_util.conv2d(net2, part_num, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net, net2, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet++, input is BxNxF, output Bxnum_class """ end_points = {} # COG l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) # Features l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, -1]) end_points['l0_xyz'] = l0_xyz # Set Abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, npoint=4096, radius_list=[0.5, 1, 2], nsample_list=[8, 16, 32], mlp_list=[[32, 32, 64], [64, 64, 128], [128, 128, 256]], is_training=is_training, pooling='max_and_avg', bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, npoint=256, radius_list=[2, 4, 8], nsample_list=[32, 64, 128], mlp_list=[[128, 128, 256], [256, 256, 512], [256, 256, 512]], is_training=is_training, pooling='max_and_avg', bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points = pointnet_sa_module_msg(l2_xyz, l2_points, npoint=32, radius_list=[4, 8, 16], nsample_list=[64, 64, 128], mlp_list=[[256, 256, 512], [512, 512, 1024], [512, 1024, 1024]], is_training=is_training, pooling='max_and_avg', bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512, 512], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def multi_modal(point_cloud, view_images, is_training=False, bn_decay=None, n_classes=40, get_ft=False, get_mask=False): """ :param point_cloud: (B, N, 3) :param view_images: (B, V, W, H, C) :param is_training: is_training for dropout and bn :param bn_decay: bn_decay :param n_classes: 40 :return: multi-modal logit and mvcnn logit """ fc6_b = MVCNN.inference_multiview(view_images, n_classes, is_training=is_training) mv_global = tf_util.fully_connected(fc6_b, 1024, bn=True, is_training=is_training, scope='pc_mv_t', bn_decay=bn_decay) batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value k = 20 point_cloud_transformed = trans_net(point_cloud, k=k, is_training=is_training, bn_decay=bn_decay, scope='pc_transform_net1') adj_matrix = tf_util.pairwise_distance(point_cloud_transformed) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(point_cloud_transformed, nn_idx=nn_idx, k=k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='pc1', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net1 = net adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='pc2', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net2 = net net = residual_attn_block(net, mv_global, k=k, C_out=64, get_mask=get_mask, C_attn=256, is_training=is_training, bn_decay=bn_decay, scope='pc_attn_1') if get_mask: net3, mask1 = net net = net3 else: net3 = net net = residual_attn_block(net, mv_global, k=k, C_out=128, get_mask=get_mask, C_attn=256, is_training=is_training, bn_decay=bn_decay, scope='pc_attn_2') if get_mask: net4, mask2 = net net = net3 else: net4 = net net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='pc_agg', bn_decay=bn_decay) net = tf.reduce_max(net, axis=1, keep_dims=True) # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf.concat([net, mv_global], axis=-1) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='pc_fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='pc_dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='pc_fc2', bn_decay=bn_decay) ft = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='pc_dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='pc_fc3') if get_ft: return net, ft elif get_mask: return net, mask1, mask2 else: return net
def get_model(inputs, is_training, bn_decay=None, num_class=40, FLAGS=None): """ Classification PointNet, input is BxNx3, output Bx40 """ point_cloud = inputs[:, :, 0:3] if FLAGS.normal: D = 6 points = inputs[:, :, 3:] else: D = 3 points = None # --------------------------------------- STN ------------------------------------- if FLAGS.STN: with tf.variable_scope('transform_net') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud = tf.matmul(point_cloud, transform) # ---------------------------------- Node Sampling -------------------------------- with tf.variable_scope('group_sampling') as sc: KNN = FLAGS.KNN point_cloud_sampled, nn_points, _, _ = sample_and_group( npoint=FLAGS.node_num, radius=0.2, nsample=KNN, xyz=point_cloud, points=points, knn=True, use_xyz=True) point_cloud_sampled = tf.expand_dims(point_cloud_sampled, axis=-1) net1 = tf_util.conv2d(point_cloud_sampled, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1_1', bn_decay=bn_decay) net1 = tf.tile(net1, multiples=[1, 1, KNN, 1]) net1 = tf.expand_dims(net1, axis=-2) nn_points = tf.expand_dims(nn_points, axis=-1) net = tf_util.conv3d(nn_points, 64, [1, 1, D], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv1_2', bn_decay=bn_decay) concat = tf.concat(values=[net, net1], axis=-1) net = tf_util.conv3d(concat, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv3d(net, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) # ---------------------- local pooling: merge local feature ------------------------- if FLAGS.local_pool == 'average': pool_k = tf_util.avg_pool3d(net, kernel_size=[1, KNN, 1], stride=[1, 2, 2], padding='VALID', scope='pool_k') else: pool_k = tf_util.max_pool3d(net, kernel_size=[1, KNN, 1], stride=[1, 2, 2], padding='VALID', scope='pool_k') net = tf.squeeze(pool_k, axis=2) # ---------------------------------- VLAD layer -------------------------------------- net, index = VLAD(net, FLAGS, is_training, bn_decay, layer_name='VLAD') # -------------------------------- classification ------------------------------------ with tf.name_scope('fc_layer'): net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, num_class, activation_fn=None, scope='fc3') return net, index
def pointnet(self, point_cloud, is_training, bn=True, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bxn where n is num classes """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.layers.flatten(net) net = tf_util.fully_connected(net, 512, bn=bn, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=bn, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, self.n_classes, activation_fn=None, scope='fc3') return net
def get_model(inputs, cls_label, is_training=None, bn_decay=None, NUM_CATEGORIES=16): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = inputs.get_shape()[0].value num_point = inputs.get_shape()[1].value point_cloud = inputs[:, :, 0:3] points = inputs[:, :, 3:] with tf.variable_scope('transform_net') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) with tf.variable_scope('group_sampling') as sc: KNN = 16 Node_NUM = 256 point_cloud_sampled, nn_points, _, _ = sample_and_group(npoint=Node_NUM, radius=0.2, nsample=KNN, xyz=point_cloud_transformed, points=points, knn=True, use_xyz=True) net1 = tf_util.conv2d(tf.expand_dims(point_cloud_sampled, axis=-1), 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1_1', bn_decay=bn_decay) net1 = tf.tile(net1, multiples=[1, 1, KNN, 1]) net1 = tf.expand_dims(net1, axis=-2) nn_points = tf.expand_dims(nn_points, axis=-1) net = tf_util.conv3d(nn_points, 64, [1, 1, 6], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv1_2', bn_decay=bn_decay) concat = tf.concat(values=[net, net1], axis=-1) net = tf_util.conv3d(concat, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv3d(net, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) pool_k = tf_util.max_pool3d(net, kernel_size=[1, KNN, 1], stride=[1, 2, 2], padding='VALID', scope='pool_k') net = tf.squeeze(pool_k, axis=2) # vlad layer vlad_out, index = VLAD_part(net, 50, is_training, bn_decay, layer_name='VLAD') vlad_out = tf_util.conv2d(vlad_out, 384, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='vlad_conv3', bn_decay=bn_decay) vlad_out = tf_util.conv2d(vlad_out, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='vlad_conv4', bn_decay=bn_decay) out_max = tf.nn.max_pool(vlad_out, ksize=[1, Node_NUM, 1, 1], strides=[1, 2, 2, 1], padding='VALID') expand = tf.tile(out_max, multiples=[1,Node_NUM,1,1]) concat = tf.concat([expand, vlad_out, net], axis=-1) concat = tf_util.conv2d(concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) concat = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) concat = tf.squeeze(concat, axis=2) # segmentation network cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1]) l0_points = pointnet_fp_module(xyz1=point_cloud_transformed, xyz2=point_cloud_sampled, points1=tf.concat([cls_label_one_hot, point_cloud_transformed, points], axis=-1), points2=concat, mlp=[128, 128], is_training=is_training, bn_decay=bn_decay, scope='layer6') l0_points = tf.expand_dims(l0_points, axis=2) net = tf_util.conv2d(l0_points, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 50, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='fc2') net = tf.squeeze(net, axis=2) return net, index
def get_model_other(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3 and BxNx1, output Bx4 """ batch_size = point_cloud.get_shape()[0] # .value num_point = point_cloud.get_shape()[1] # .value end_points = {} # transform net for input x,y,z with tf.compat.v1.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud[:, :, 1:4], is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud[:, :, 1:4], transform) input_image = tf.expand_dims(point_cloud_transformed, -1) # First MLP layers net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) # transform net for the features with tf.compat.v1.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) # point_feat = tf.expand_dims(net_transformed, [2]) # add the additional features to the second MLP layers # point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0:1], [2]) concat_other = tf.concat(axis=2, values=[ point_cloud[:, :, 0:1], net_transformed, point_cloud[:, :, 4:para.dim] ]) concat_other = tf.expand_dims(concat_other, [2]) # second MLP layers net = tf_util.conv2d( concat_other, 64, [1, 1], # 64 is the output #neuron padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 4, activation_fn=None, scope='fc3') return net, end_points
def pct_model(point_cloud, is_training, bn_decay=None): """ :param point_cloud: :param is_training: :param bn_decay: :return: """ # point_cloud -> [batch_size, num_point, 3] batch_size = point_cloud.get_shape()[0].value point_dim = point_cloud.get_shape()[2].value """Input Embedding module""" # [batch_size, num_point, 64] x = tf_util.conv1d(point_cloud, 64, kernel_size=1, padding='VALID', stride=1, bn=True, is_biases=False, is_training=is_training, scope='conv0', bn_decay=bn_decay) x = tf_util.conv1d(x, 64, kernel_size=1, padding='VALID', stride=1, bn=True, is_biases=False, is_training=is_training, scope='conv1', bn_decay=bn_decay) """ Sample and Group """ new_xyz, new_feature, _, _ = pointnet_util.sample_and_group( npoint=512, radius=0.15, nsample=32, xyz=point_cloud, points=x, knn=True, use_xyz=True) # print(new_xyz.shape) # print("new_feature.shape", new_feature.shape) feature_0 = local_op(new_feature, out_dim=128, scope="SG1", bn_decay=bn_decay, is_training=is_training) new_xyz, new_feature, _, _ = pointnet_util.sample_and_group( npoint=256, radius=0.2, nsample=32, xyz=new_xyz, points=feature_0, knn=True, use_xyz=True) # NHC feature_1 = local_op(new_feature, out_dim=256, scope="SG2", bn_decay=bn_decay, is_training=is_training) # # NHC x = pt_last(feature_1, scope="pct_layer", out_dim=256, bn_decay=bn_decay, is_training=is_training) # concat in C (NHC) axis x = tf.concat([x, feature_1], axis=-1) x = tf_util.conv1d(x, 1024, kernel_size=1, padding='VALID', stride=1, bn=True, is_biases=False, is_training=is_training, scope='conv2', bn_decay=bn_decay, activation_fn=None) x = tf.nn.leaky_relu(x, alpha=0.2) x = tf.reduce_max(x, axis=1) """ ++++++++++++++++++++++++++++++++++++++++ Decoder ++++++++++++++++++++++++++++++++++++++++ """ x = tf_util.fully_connected(x, 512, bn=True, is_training=is_training, is_biases=False, scope='fc1', bn_decay=bn_decay) x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp1') x = tf_util.fully_connected(x, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp2') x = tf_util.fully_connected(x, 40, activation_fn=None, scope='fc3') return x
def get_model_other(point_cloud, is_training, bn_decay=None): """ PointNet2 with multi-scale grouping Semantic segmentation network that uses feature propogation layers """ batch_size = point_cloud.get_shape()[0] # .value end_points = {} l0_xyz = point_cloud[:, :, 1:4] l0_points = tf.concat( axis=2, values=[point_cloud[:, :, 0:1], point_cloud[:, :, 4:para.dim]]) # Set abstraction layers # input B 1024 1 3 => 64+128+128 = 320 max pooling in small group n = 16 32 128 l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') # , use_nchw=True # input B 512 320 => 128+256+256 = 640 max pooling in small group n = 32 64 128 l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') # input B 128 640 => 1024, max pooling in all pointcloud = 128 # MLP layer to gather 3 scale features _, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # input B 1 1024 # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') return net, end_points
def get_model_other(point_cloud, is_training, bn_decay=None): """ DesnsePoint with 2 PPools + 3 PConvs + 1 global pooling narrowness k = 24; group number g = 2 """ batch_size = point_cloud.get_shape()[0] # .value end_points = {} # concatenate all features together SF,x,y,z, distance minSF point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1) l0_xyz = point_cloud[:, :, 1:4] l0_points = tf.concat( axis=2, values=[point_cloud_SF, point_cloud[:, :, 4:para.dim]]) # first stage: 1 PPool, 3 EnhancedPConv all_xyz, all_points = densepoint_module(l0_xyz, l0_points, is_training, bn_decay, npoint=512, radius=0.25, nsample=64, mlp=para.k_add * 4, scope='PPool1', ppool=True) # second stage: 2 PPool, 3 EnhancedPConv all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, npoint=256, radius=0.27, nsample=64, mlp=para.k_add * 4 - 3, scope='PPool2', ppool=True) for i in range(4): # B 128 1 93 -> 24 all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, npoint=256, radius=0.32, nsample=16, mlp=para.k_add * 4, group_num=para.group_num, scope=f'PConv1_{i + 1}') # # second stage: 2 PPool, 3 EnhancedPConv # all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, # npoint=128, radius=0.32, nsample=64, mlp=para.k_add * 4 - 3, # scope='PPool3', ppool=True) # # for i in range(4): # B 128 1 93 -> 24 # all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, # npoint=128, radius=0.39, nsample=16, mlp=para.k_add * 4, # group_num=para.group_num, # scope=f'PConv2_{i + 1}') l3_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, mlp=512, scope='GloPool') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, part_num, batch_size, \ num_point, weight_decay, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} with tf.variable_scope('transform_net1') as sc: K = 3 transform = get_transform(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) out1 = tf_util.conv2d(input_image, 64, [1, K], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out3 = tf_util.conv2d(out2, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128]) net_transformed = tf.matmul(squeezed_out3, transform) net_transformed = tf.expand_dims(net_transformed, [2]) out4 = tf_util.conv2d(net_transformed, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point, 1], padding='VALID', scope='maxpool') expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5]) net = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay) net = tf_util.dropout(net, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay) net = tf_util.dropout(net, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay) net = tf_util.conv2d(net, part_num, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay) net = tf.reshape(net, [batch_size, num_point, part_num]) return net, end_points
def get_model(point_cloud, is_training, normals, use_local_frame=True, add_normals=False, bn=True, bn_decay=None, use_xavier=True, align_pointclouds=False, drop_prob=0.5, n_classes=1, k=20): """ Part segmentation DGCNN, input is BxNxnFeatures, output BxnClasses """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value # Input xyz coordinates input_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) # Add tangent vectors and normals for local frames calculation local_frame_data = None if use_local_frame: tangent_vec1 = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3], name="tangent_vec1") tangent_vec2 = tf.slice(point_cloud, [0, 0, 6], [-1, -1, 3], name="tangent_vec2") local_frame_data = (tangent_vec1, tangent_vec2, normals) # Point clouds global alignment if align_pointclouds: # Calculate pairwise distance on global coordinates and find k-nn's for each point adj = tf_util.pairwise_distance(input_xyz) nn_idx = tf_util.knn(adj, k=k) input_xyz = tf.expand_dims(input_xyz, -1) edge_feature = tf_util.get_edge_feature(input_xyz, nn_idx=nn_idx, k=k) with tf.variable_scope('transform_net_global') as sc: global_transform = global_spatial_transformer( point_cloud=edge_feature, is_training=is_training, bn=bn, bn_decay=bn_decay, is_dist=True) input_xyz = tf.matmul(tf.squeeze(input_xyz, axis=-1), global_transform) if add_normals: if input_xyz.shape.ndims == 4: input_xyz = tf.squeeze(input_xyz, axis=-1) input_xyz = tf.concat([input_xyz, normals], axis=-1) input_image = tf.expand_dims(input_xyz, -1) adj = tf_util.pairwise_distance(input_xyz) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k, use_local_frame=use_local_frame, local_frame_data=local_frame_data, add_normals=add_normals) # EdgeConv layer 1 {64, 64} out1 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='adj_conv1', bn_decay=bn_decay, is_dist=True, use_xavier=use_xavier) out2 = tf_util.conv2d(out1, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='adj_conv2', bn_decay=bn_decay, is_dist=True, use_xavier=use_xavier) net_1 = tf.reduce_max(out2, axis=-2, keep_dims=True) # EdgeConv layer 2 {64, 64} adj = tf_util.pairwise_distance(net_1) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k) out3 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='adj_conv3', bn_decay=bn_decay, is_dist=True, use_xavier=use_xavier) out4 = tf_util.conv2d(out3, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='adj_conv4', bn_decay=bn_decay, is_dist=True, use_xavier=use_xavier) net_2 = tf.reduce_max(out4, axis=-2, keep_dims=True) # EdgeConv layer 3 {64} adj = tf_util.pairwise_distance(net_2) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k) out5 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='adj_conv5', bn_decay=bn_decay, is_dist=True, use_xavier=use_xavier) net_3 = tf.reduce_max(out5, axis=-2, keep_dims=True) # [EdgeConv1, EdgeConv2, EdgeConv3] -> MLP {64} out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='adj_conv7', bn_decay=bn_decay, is_dist=True, use_xavier=use_xavier) out_max = tf_util.max_pool2d(out7, [num_point, 1], padding='VALID', scope='maxpool') expand = tf.tile(out_max, [1, num_point, 1, 1]) # Concat [global_feature, EdgeConv1, EdgeConv2, EdgeConv3] concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3]) # FC layer - MLP{256, 256, 128, n_classes} net2 = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=bn, is_training=is_training, scope='seg/conv1', is_dist=True, use_xavier=use_xavier) net2 = tf_util.dropout(net2, keep_prob=1 - drop_prob, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=bn, is_training=is_training, scope='seg/conv2', is_dist=True, use_xavier=use_xavier) net2 = tf_util.dropout(net2, keep_prob=1 - drop_prob, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=bn, is_training=is_training, scope='seg/conv3', is_dist=True, use_xavier=use_xavier) net2 = tf_util.conv2d(net2, n_classes, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv4', is_dist=False, use_xavier=use_xavier) net2 = tf.reshape(net2, [batch_size, num_point, n_classes]) return net2
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is BxNxF, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # CONV layers net = tf_util.conv2d(input_image, 64, [1, 9], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) points_feat1 = tf_util.conv2d(net, 2048, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # MAX Pooling pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point, 1], padding='VALID', scope='maxpool1') # FC layers pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1]) pc_feat1 = tf_util.fully_connected(pc_feat1, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) # CONCAT pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1]) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) # CONV Layers net = tf_util.conv2d(points_feat1_concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6') net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7') net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv8') net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv9') net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, num_class, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv10') net = tf.squeeze(net, [2]) return net, end_points
def get_model(point_cloud, up_ratio, is_training, bradius=1.0, knn=30, scope='generator', weight_decay=0.0, bn_decay=None, bn=True, fd=64, fD=1024): with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc: batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_point_cloud = tf.expand_dims(point_cloud, -1) adj = tf_util.pairwise_distance(point_cloud) nn_idx = tf_util.knn(adj, k=knn) edge_feature = tf_util.get_edge_feature(input_point_cloud, nn_idx=nn_idx, k=knn) with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(edge_feature, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_point_cloud = tf.expand_dims(point_cloud_transformed, -1) adj = tf_util.pairwise_distance(point_cloud_transformed) nn_idx = tf_util.knn(adj, k=knn) edge_feature = tf_util.get_edge_feature(input_point_cloud, nn_idx=nn_idx, k=knn) out1 = tf_util.conv2d(edge_feature, fd, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, weight_decay=weight_decay, scope='dgcnn_conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, fd, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, weight_decay=weight_decay, scope='dgcnn_conv2', bn_decay=bn_decay) net_max_1 = tf.reduce_max(out2, axis=-2, keep_dims=True) net_mean_1 = tf.reduce_mean(out2, axis=-2, keep_dims=True) out3 = tf_util.conv2d(tf.concat([net_max_1, net_mean_1], axis=-1), fd, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, weight_decay=weight_decay, scope='dgcnn_conv3', bn_decay=bn_decay) adj = tf_util.pairwise_distance(tf.squeeze(out3, axis=-2)) nn_idx = tf_util.knn(adj, k=knn) edge_feature = tf_util.get_edge_feature(out3, nn_idx=nn_idx, k=knn) out4 = tf_util.conv2d(edge_feature, fd, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, weight_decay=weight_decay, scope='dgcnn_conv4', bn_decay=bn_decay) net_max_2 = tf.reduce_max(out4, axis=-2, keep_dims=True) net_mean_2 = tf.reduce_mean(out4, axis=-2, keep_dims=True) out5 = tf_util.conv2d(tf.concat([net_max_2, net_mean_2], axis=-1), fd, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, weight_decay=weight_decay, scope='dgcnn_conv5', bn_decay=bn_decay) adj = tf_util.pairwise_distance(tf.squeeze(out5, axis=-2)) nn_idx = tf_util.knn(adj, k=knn) edge_feature = tf_util.get_edge_feature(out5, nn_idx=nn_idx, k=knn) out6 = tf_util.conv2d(edge_feature, fd, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, weight_decay=weight_decay, scope='dgcnn_conv6', bn_decay=bn_decay) net_max_3 = tf.reduce_max(out6, axis=-2, keep_dims=True) net_mean_3 = tf.reduce_mean(out6, axis=-2, keep_dims=True) out7 = tf_util.conv2d(tf.concat([net_max_3, net_mean_3], axis=-1), fd, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, weight_decay=weight_decay, scope='dgcnn_conv7', bn_decay=bn_decay) out8 = tf_util.conv2d(tf.concat([out3, out5, out7], axis=-1), fD, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='dgcnn_conv8', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out8, [num_point, 1], padding='VALID', scope='maxpool') expand = tf.tile(out_max, [1, num_point, 1, 1]) concat_unweight = tf.concat(axis=3, values=[ expand, net_max_1, net_mean_1, out3, net_max_2, net_mean_2, out5, net_max_3, net_mean_3, out7, out8 ]) feat_list = [ "expand", "net_max_1", "net_mean_1", "out3", "net_max_2", "net_mean_2", "out5", "net_max_3", "net_mean_3", "out7", "out8" ] out_attention = tf_util.conv2d(concat_unweight, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='attention_conv1', weight_decay=weight_decay) out_attention = tf_util.conv2d(out_attention, 64, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='attention_conv2', weight_decay=weight_decay) out_attention = tf_util.conv2d(out_attention, len(feat_list), [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='attention_conv3', weight_decay=weight_decay) out_attention = tf_util.max_pool2d(out_attention, [num_point, 1], padding='VALID', scope='attention_maxpool') out_attention = tf.nn.softmax(out_attention) tmp_attention = tf.squeeze(out_attention) for i in range(len(feat_list)): tmp1 = tf.slice(out_attention, [0, 0, 0, i], [1, 1, 1, 1]) exec('dim = %s.get_shape()[-1].value' % feat_list[i]) tmp2 = tf.tile(tmp1, [1, 1, 1, dim]) if i == 0: attention_weight = tmp2 else: attention_weight = tf.concat([attention_weight, tmp2], axis=-1) attention_weight = tf.tile(attention_weight, [1, num_point, 1, 1]) concat = tf.multiply(concat_unweight, attention_weight) concat = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='concat_conv', weight_decay=weight_decay) concat = tf_util.dropout(concat, keep_prob=0.6, is_training=is_training, scope='dg1') with tf.variable_scope('uv_predict'): uv_2d = tf_util.conv2d(concat, up_ratio * 2, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, activation_fn=None, bn=None, is_training=is_training, scope='uv_conv1', weight_decay=weight_decay) uv_2d = tf.reshape(uv_2d, [batch_size, num_point, up_ratio, 2]) uv_2d = tf.concat( [uv_2d, tf.zeros([batch_size, num_point, up_ratio, 1])], axis=-1) with tf.variable_scope('T_predict'): affine_T = tf_util.conv2d(concat, 9, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, activation_fn=None, bn=None, is_training=is_training, scope='patch_conv1', weight_decay=weight_decay) affine_T = tf.reshape(affine_T, [batch_size, num_point, 3, 3]) uv_3d = tf.matmul(uv_2d, affine_T) uv_3d = uv_3d + tf.tile(tf.expand_dims(point_cloud, axis=-2), [1, 1, up_ratio, 1]) uv_3d = tf.transpose(uv_3d, perm=[0, 2, 1, 3]) uv_3d = tf.reshape(uv_3d, [batch_size, num_point * up_ratio, -1]) with tf.variable_scope('normal_predict'): dense_normal_offset = tf_util.conv2d(concat, up_ratio * 3, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, activation_fn=None, bn=None, is_training=is_training, scope='normal_offset_conv1', weight_decay=weight_decay) dense_normal_offset = tf.reshape( dense_normal_offset, [batch_size, num_point, up_ratio, 3]) sparse_normal = tf.convert_to_tensor([0, 0, 1], dtype=tf.float32) sparse_normal = tf.expand_dims(sparse_normal, axis=0) sparse_normal = tf.expand_dims(sparse_normal, axis=0) sparse_normal = tf.expand_dims(sparse_normal, axis=0) sparse_normal = tf.tile(sparse_normal, [batch_size, num_point, 1, 1]) sparse_normal = tf.matmul(sparse_normal, affine_T) sparse_normal = tf.nn.l2_normalize(sparse_normal, axis=-1) dense_normal = tf.tile(sparse_normal, [1, 1, up_ratio, 1]) + dense_normal_offset dense_normal = tf.nn.l2_normalize(dense_normal, axis=-1) dense_normal = tf.transpose(dense_normal, perm=[0, 2, 1, 3]) dense_normal = tf.reshape(dense_normal, [batch_size, num_point * up_ratio, -1]) with tf.variable_scope('up_layer'): if not np.isscalar(bradius): bradius_expand = tf.expand_dims(tf.expand_dims(bradius, axis=-1), axis=-1) else: bradius_expand = bradius bradius_expand = bradius_expand grid = tf.expand_dims(uv_3d * bradius_expand, axis=2) concat_up = tf.tile(concat, (1, up_ratio, 1, 1)) concat_up = tf.concat([concat_up, grid], axis=-1) concat_up = tf_util.conv2d(concat_up, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='up_layer1', bn_decay=bn_decay, weight_decay=weight_decay) concat_up = tf_util.dropout(concat_up, keep_prob=0.6, is_training=is_training, scope='up_dg1') concat_up = tf_util.conv2d(concat_up, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='up_layer2', bn_decay=bn_decay, weight_decay=weight_decay) concat_up = tf_util.dropout(concat_up, keep_prob=0.6, is_training=is_training, scope='up_dg2') # get xyz coord_z = tf_util.conv2d(concat_up, 1, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, is_training=is_training, scope='fc_layer', weight_decay=weight_decay) coord_z = tf.reshape(coord_z, [batch_size, up_ratio, num_point, 1]) coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3]) coord_z = tf.concat( [tf.zeros_like(coord_z), tf.zeros_like(coord_z), coord_z], axis=-1) coord_z = tf.matmul(coord_z, affine_T) coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3]) coord_z = tf.reshape(coord_z, [batch_size, num_point * up_ratio, -1]) coord = uv_3d + coord_z return coord, dense_normal, tf.squeeze(sparse_normal, [2])
def get_model_other(point_cloud, is_training, bn_decay=None): """ B: batch size; N: number of points, C: channels; k: number of nearest neighbors point_cloud: B*N*C """ end_points = {} minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1)) batch_size = point_cloud.get_shape()[0] # .value # # 1. graph for first EdgeConv B N C=6 adj_matrix = tf_util.pairwise_distance( point_cloud[:, :, :para.dim]) # B N C=6 => B*N*N # adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:para.dim]) # B N C=6 => B*N*N nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn1'] = allSF_dist point_cloud = tf.expand_dims(point_cloud[:, :, :para.dim], axis=-2) # point_cloud = tf.expand_dims(point_cloud[:, :, 1:para.dim], axis=-2) edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn1', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net1 = net # # 2. graph for second EdgeConv B N C=64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn2'] = allSF_dist # net: B*N*1*6+64=71 net = tf.concat([point_cloud, net1], axis=-1) # edge_feature: B*N*k*142 edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn2', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net2 = net # 3. graph for third EdgeConv B N C=64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn3'] = allSF_dist # net: B*N*1*6+64+64=134 net = tf.concat([point_cloud, net1, net2], axis=-1) # edge_feature: B*N*k*268 edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn3', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net3 = net # 4. graph for fourth EdgeConv B N C=64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn4'] = allSF_dist # net: B*N*1*6+64+64+64=198 net = tf.concat([point_cloud, net1, net2, net3], axis=-1) # edge_feature: B*N*k*396 edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn4', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net4 = net # input: B*N*1*6+64+64+128+128 = 326 => net: B*N*1*1024 net = tf_util.conv2d(tf.concat([point_cloud, net1, net2, net3, net4], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='agg', bn_decay=bn_decay) # net: B*1*1*1024 # SF_features = tf.gather(net, indices=minSF, axis=1, batch_dims=1) net = tf.reduce_max(net, axis=1, keepdims=True) # SF_all = tf.concat([SF_features,net], axis=-1) # net: B*1024 net = tf.squeeze(net) # net: B*2048 # net = tf.squeeze(SF_all) # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) print(net.get_shape()) end_points['global_feature'] = net # Fully connected end_points: classifier net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['fc1'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) end_points['fc2'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') end_points['fc3'] = net return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ with tf.variable_scope('transform_net') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) # KNN local search #knn_point = KNN_search(point_cloud_transformed, KNN=KNN, name_scope='KNN_search') # 32 x 1024 x KNN x 6 #knn_point = tf.expand_dims(knn_point, axis=-1) # 32 x 1024 x KNN x 3 x 1 with tf.variable_scope('group_sampling') as sc: KNN = 16 point_cloud_transformed, _, _, nn_points = sample_and_group( npoint=1024, radius=0.1, nsample=KNN, xyz=point_cloud_transformed, points=None, knn=True, use_xyz=False) point_cloud_transformed = tf.expand_dims(point_cloud_transformed, axis=-1) net1 = tf_util.conv2d(point_cloud_transformed, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1_1', bn_decay=bn_decay) # 32 x 1024 x 1 x 64 net1 = tf.tile(net1, multiples=[1, 1, KNN, 1]) # 32 x 1024 x 16 x 64 net1 = tf.expand_dims(net1, axis=-2) nn_points = tf.expand_dims(nn_points, axis=-1) # 32 x 1024 x 16 x 3 x 1 net = tf_util.conv3d(nn_points, 64, [1, 1, 3], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv1_2', bn_decay=bn_decay) # 32 x 1024 x 16 x 1 x 64 concat = tf.concat(values=[net, net1], axis=-1) net = tf_util.conv3d(concat, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv3d(net, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) # local pooling: merge local feature pool_k = tf_util.max_pool3d(net, kernel_size=[1, KNN, 1], stride=[1, 2, 2], padding='VALID', scope='pool_k') # 32 x 1024 x 1 x 1 x 128 net1 = tf.squeeze(pool_k, axis=2) # VLAD layer net2, index = VLAD(net1, 16, is_training, bn_decay, layer_name='VLAD_layer1') net = VLAD_layer(net2, net1, 16, is_training, bn_decay, layer_name='VLAD_layer2') # classification with tf.name_scope('fc_layer'): net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, index