def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Set abstraction layers # Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4). # So we only use NCHW for layer 1 until this issue can be resolved. l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2') cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1]) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3]) # Set Abstraction layers l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Layer 1 l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_instance_seg_v2_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v2 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 128, [0.2,0.4,0.8], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 32, [0.4,0.8,1.6], [64,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[128,256,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2) l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [128,128], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [128,128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') logits = tf_util.conv1d(net, 2, 1, padding='VALID', activation_fn=None, scope='conv1d-fc2') return logits, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [16,32,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2,0.4,0.8], [32,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # npoints = [256, 64, 10] # npoints = [512, 128, 10] # npoints = [512, 128, 8] npoints = [512, 128, 32] ANCHOR = 27 l1_xyz, l1_points, l1_nn, l1_offset = sample_and_group(npoint=npoints[0], radius=0.2, nsample=32, xyz=l0_xyz, points=None, knn=False, use_xyz=True) l2_xyz, l2_points, l2_nn, l2_offset = sample_and_group(npoint=npoints[1], radius=0.4, nsample=64, xyz=l1_xyz, points=None, knn=False, use_xyz=True) l3_xyz, l3_points, l3_nn, l3_offset = sample_and_group(npoint=npoints[2], radius=0.8, nsample=64, xyz=l2_xyz, points=None, knn=False, use_xyz=True) # # (1024, 3) -> (1024, 64) fc-0 # # (1024, 64) -> (1024, 64) conv-1 # # (1024, 64) -> (1024, 128) fc-1 # # (1024, 128) -> (1024, 256) conv-2 # # (1024, 256) -> (1024, 256) fc-2 # k_list = [10, 10, 10] # cube_list = [0.2, 0.2, 0.2] # channels = [64, 128, 256] # xyz_list = [l0_xyz, l0_xyz, l0_xyz, l0_xyz] # # (1024, 3) -> (1024, 64) fc-0 # # (1024, 64) -> (512, 64) conv-1 # # (512, 64) -> (512, 256) fc-1 # # (512, 256) -> (10, 256) conv-2 # # (10, 1024) -> (10, 1024) fc-2 # k_list = [32, 64, 64] # cube_list = [0.1, 0.2, 0.4] # channels = [64, 256, 512] # xyz_list = [l0_xyz, l1_xyz, l2_xyz, l3_xyz] # # (1024, 3) -> (1024, 64) fc-0 # # (1024, 64) -> (512, 64) conv-1 # # (512, 64) -> (512, 256) fc-1 # # (512, 256) -> (10, 256) conv-2 # # (10, 1024) -> (10, 1024) fc-2 # k_list = [32, 64, 128] # cube_list = [0.2, 0.4, 1.0] # channels = [64, 256, 512] # xyz_list = [l0_xyz, l1_xyz, l2_xyz, l3_xyz] # # (1024, 3) -> (1024, 64) fc-0 # # (1024, 64) -> (512, 64) conv-1 # # (512, 64) -> (512, 256) fc-1 # # (512, 256) -> (10, 256) conv-2 # # (10, 1024) -> (10, 1024) fc-2 # k_list = [10, 10, 10] # cube_list = [0.1, 0.2, 0.4] # channels = [64, 256, 1024] # xyz_list = [l0_xyz, l1_xyz, l2_xyz, l3_xyz] # k_list = [10, 10, 10, 10, 10] # cube_list = [0.2, 0.2, 0.4, 0.4, 0.8] # channels = [64, 64, 256, 256, 1024] # xyz_list = [l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz] # # (1024, 3) -> (1024, 32) fc-0 # # (1024, 32) -> (1024, 32) conv-1-1 K=32 l0 -> l0 # # (1024, 32) -> (1024, 32) fc-1-1 # # (1024, 32) -> (512, 32) conv-1-2 K=32 l0 -> l1 # # (512, 32) -> (512, 128) fc-1-2 # # (512, 128) -> (512, 128) conv-2-1 K=16 l1 -> l1 # # (512, 128) -> (512, 128) fc-2-1 # # (512, 128) -> (128, 128) conv-2-2 K=16 l1 -> l2 # # (128, 128) -> (128, 512) fc-2-2 # # (128, 512) -> (128, 512) conv-3-1 K=8 l2 -> l2 # # (128, 512) -> (128, 512) fc-3-1 # # (128, 512) -> (8, 512) conv-3-2 K=8 l2 -> l3 # # (8, 512) -> (8, 512) fc 3-2 # k_list = [32, 32, 16, 16, 8, 8] # cube_list = [0.2, 0.2, 0.4, 0.4, 0.8, 0.8] # channels = [32, 32, 128, 128, 512, 512] # xyz_list = [l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz] # (1024, 3) -> (1024, 32) fc-0 # (1024, 32) -> (1024, 32) conv-1-1 K=32 l0 -> l0 # (1024, 32) -> (1024, 32) fc-1-1 # (1024, 32) -> (512, 32) conv-1-2 K=32 l0 -> l1 # (512, 32) -> (512, 128) fc-1-2 # (512, 128) -> (512, 128) conv-2-1 K=16 l1 -> l1 # (512, 128) -> (512, 128) fc-2-1 # (512, 128) -> (128, 128) conv-2-2 K=16 l1 -> l2 # (128, 128) -> (128, 512) fc-2-2 # (128, 512) -> (128, 512) conv-3-1 K=16 l2 -> l2 # (128, 512) -> (128, 512) fc-3-1 # (128, 512) -> (1, 512) conv-3-2 K=128 l2 -> l3 # (1, 512) -> (1, 512) fc 3-2 # k_list = [32, 32, 16, 16, 16, 128] # cube_list = [0.2, 0.2, 0.4, 0.4, 0.8, 1.2] # channels = [32, 32, 128, 128, 512, 512] # xyz_list = [l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz] k_list = [32, 32, 16, 16, 16, 16] cube_list = [0.2, 0.2, 0.4, 0.4, 0.8, 0.8] channels = [32, 32, 128, 128, 512, 512] xyz_list = [l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz] # # (1024, 3) -> (1024, 32) fc-0 # # (1024, 32) -> (1024, 32) conv-1-1 K=10 l0 -> l0 # # (1024, 32) -> (1024, 32) fc-1-1 # # (1024, 32) -> (1024, 32) conv-1-2 K=10 l0 -> l0 # # (1024, 32) -> (1024, 32) fc-1-2 # # (1024, 32) -> (1024, 32) conv-1-3 K=10 l0 -> l0 # # (1024, 32) -> (1024, 32) fc-1-3 # # (1024, 32) -> (512, 32) conv-1-4 K=10 l0 -> l1 # # (512, 32) -> (512, 128) fc-1-4 # # (512, 128) -> (512, 128) conv-2-1 K=10 l1 -> l1 # # (512, 128) -> (512, 128) fc-2-1 # # (512, 128) -> (128, 128) conv-2-2 K=10 l1 -> l2 # # (128, 128) -> (128, 512) fc-2-2 # # (128, 512) -> (128, 512) conv-3-1 K=10 l2 -> l2 # # (128, 512) -> (128, 512) fc-3-1 # # (128, 512) -> (10, 512) conv-3-2 K=10 l2 -> l3 # # (10, 512) -> (10, 512) fc 3-2 # k_list = [10, 10, 10, 10, 10, 10, 10, 10] # cube_list = [0.2, 0.2, 0.2, 0.2, 0.4, 0.4, 0.8, 0.8] # channels = [32, 32, 32, 32, 128, 128, 512, 512] # xyz_list = [l0_xyz, l0_xyz, l0_xyz, l0_xyz, l1_xyz, l1_xyz, l2_xyz, l2_xyz, l3_xyz] nlayers = len(xyz_list) - 1 if True: offset_list = [ tf.py_func(get_knn, [_xyz, _xyz2, _k], [tf.float32, tf.int64]) for _xyz, _xyz2, _k in zip(xyz_list[:-1], xyz_list[1:], k_list) ] nn_list = [_offset[1] for _offset in offset_list] offset_list = [_offset[0] for _offset in offset_list] offset00, _ = tf.py_func(get_knn, [xyz_list[0], xyz_list[0], k_list[0]], [tf.float32, tf.int64]) else: offset_list = [l1_offset, l2_offset, l3_offset] nn_list = [l1_nn, l2_nn, l3_nn] nn_list = [tf.cast(nn, tf.int64) for nn in nn_list] alpha_list = [ cont_filter_interp(_offset, _cube, ANCHOR) for (_offset, _cube) in zip(offset_list, cube_list) ] def _fc(points, w, bn=True): N = points.shape[0] in_channels = w.shape[0] out_channels = w.shape[1] points_ = tf.reshape(points, [-1, in_channels]) points_ = tf.matmul(points_, w) points = tf.reshape(points_, [N, -1, out_channels]) if bn: points = tf.contrib.layers.batch_norm(points, center=True, scale=True, is_training=is_training, decay=bn_decay, updates_collections=None) return points def _conv_block(points, knn, alpha, c_in, c_out): conv_init = tf.initializers.uniform_unit_scaling(1.0) conv_w = tf.get_variable('convw', [ ANCHOR, c_in, ], initializer=conv_init, trainable=True) fc_w = tf.get_variable('fcw', [c_in, c_out], initializer=tf.initializers.truncated_normal( 0.0, 2.0 / np.sqrt(float(c_out))), trainable=True) points = cont_filter_conv(points, knn, alpha, conv_w) points = _fc(points, fc_w, bn=True) points = tf.nn.relu(points) return points c_in = channels[0] c_init = 3 # points = tf.ones_like(l0_xyz[:, :, :1]) # points = tf.reduce_mean(offset00, [2]) # [B, M, K, D] # points = tf.reshape(points, [16, 1024, 3]) points = l0_xyz fc0 = tf.get_variable('w0', [c_init, c_in], initializer=tf.initializers.truncated_normal( 0.0, 2.0 / np.sqrt(float(c_in)))) points = _fc(points, fc0, bn=False) print(points) for layer in range(nlayers): with tf.variable_scope('l{}'.format(layer)): print(layer, len(alpha_list)) alpha = alpha_list[layer] nn = nn_list[layer] c_out = channels[layer] points = _conv_block(points, nn, alpha, c_in, c_out) c_in = c_out # Fully connected layers l3_points = tf.reduce_max(points, [1]) # [B, C] # l3_points = tf.reduce_mean(points, [1]) # [B, C] net = tf.reshape(l3_points, [batch_size, channels[-1]]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \ batch_size, num_point, weight_decay, graphnum, featnum, bn_decay=None): batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) k = 30 adj = tf_util.pairwise_distance(point_cloud) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k) with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(edge_feature, is_training, bn_decay, K=3, is_dist=True) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -2) adj = tf_util.pairwise_distance(point_cloud_transformed) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k) out1 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv1', bn_decay=bn_decay, is_dist=True) out2 = tf_util.conv2d(out1, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv2', bn_decay=bn_decay, is_dist=True) net_max_1 = tf.reduce_max(out2, axis=-2, keep_dims=True) net_mean_1 = tf.reduce_mean(out2, axis=-2, keep_dims=True) out3 = tf_util.conv2d(tf.concat([net_max_1, net_mean_1], axis=-1), 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv3', bn_decay=bn_decay, is_dist=True) out3A, net_max_1A, net_mean_1A = tf_util.offset_deform( input_image, out3, scope="trans_conv0", num_neighbor=k, num_graph=graphnum[0], num_feat=featnum[0], weight_decay=weight_decay, is_training=is_training, bn_decay=bn_decay) out5, net_max_2, net_mean_2 = tf_util.offset_deform( input_image, out3A, scope="trans_conv1", num_neighbor=k, num_graph=graphnum[0], num_feat=featnum[0], weight_decay=weight_decay, is_training=is_training, bn_decay=bn_decay) out7, net_max_3, net_mean_3 = tf_util.offset_deform( input_image, out5, scope="trans_conv2", num_neighbor=k, num_graph=graphnum[1], num_feat=featnum[1], weight_decay=weight_decay, is_training=is_training, bn_decay=bn_decay) '''adj = tf_util.pairwise_distance(tf.squeeze(trans2, axis=-2)) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(tf.concat([out5,trans2], axis = -1), nn_idx=nn_idx, k=k) out6 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv6', bn_decay=bn_decay, is_dist=True) net_max_3 = tf.reduce_max(out6, axis=-2, keep_dims=True) net_mean_3 = tf.reduce_mean(out6, axis=-2, keep_dims=True) out7 = tf_util.conv2d(tf.concat([net_max_3, net_mean_3], axis=-1), 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv7', bn_decay=bn_decay, is_dist=True)''' out8 = tf_util.conv2d(tf.concat([out3, out5, out7], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='adj_conv13', bn_decay=bn_decay, is_dist=True) out_max = tf_util.max_pool2d(out8, [num_point, 1], padding='VALID', scope='maxpool') one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) one_hot_label_expand = tf_util.conv2d(one_hot_label_expand, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='one_hot_label_expand', bn_decay=bn_decay, is_dist=True) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[ expand, net_max_1, net_mean_1, out3, net_max_2, net_mean_2, out5, net_max_3, net_mean_3, out7, out8 ]) net2 = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay, is_dist=True) net2 = tf_util.dropout(net2, keep_prob=0.6, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay, is_dist=True) net2 = tf_util.dropout(net2, keep_prob=0.6, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay, is_dist=True) net2 = tf_util.conv2d(net2, part_num, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay, is_dist=True) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net2
def get_scene_model(point_cloud, is_training, num_class, sigma, bn_decay=None, weight_decay=None): """ Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud[:, :, :3] l0_points = point_cloud l0_points_xyz = point_cloud[:, :, :3] l0_points_rgb = point_cloud[:, :, 3:] # Feature encoding layers l1_xyz, l1_points_xyz = feature_encoding_layer(l0_xyz, l0_points_xyz, npoint=2048, radius=0.1, sigma=sigma, K=8, mlp=[32, 32, 32], akc_channel=3, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer1_xyz') _, l1_points_rgb = feature_encoding_layer_extra(l0_xyz, l1_xyz, l0_points_rgb, npoint=2048, radius=0.1, sigma=sigma, K=8, mlp=[32, 32, 32], akc_channel=3, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer1_rgb') l1_points = tf.concat([l1_points_xyz, l1_points_rgb], -1) l1_xyz_1024, l1_points_1024 = feature_encoding_layer( l1_xyz, l1_points, npoint=1024, radius=0.1, sigma=sigma, K=8, mlp=[32, 32, 64], akc_channel=16, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer1_1024') l1_xyz_512, l1_points_512 = feature_encoding_layer( l1_xyz_1024, l1_points_1024, npoint=512, radius=0.1, sigma=sigma, K=8, mlp=[64, 64, 64], akc_channel=32, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer1_512') l2_xyz, l2_points = feature_encoding_layer(l1_xyz_512, l1_points_512, npoint=256, radius=0.2, sigma=2 * sigma, K=8, mlp=[64, 64, 128], akc_channel=32, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer2') l2_xyz_128, l2_points_128 = feature_encoding_layer( l2_xyz, l2_points, npoint=128, radius=0.2, sigma=2 * sigma, K=8, mlp=[128, 128, 128], akc_channel=None, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer2_128') l3_xyz, l3_points = feature_encoding_layer(l2_xyz_128, l2_points_128, npoint=64, radius=0.4, sigma=4 * sigma, K=8, mlp=[128, 128, 256], akc_channel=None, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer3') l4_xyz, l4_points = feature_encoding_layer(l3_xyz, l3_points, npoint=36, radius=0.8, sigma=8 * sigma, K=8, mlp=[256, 256, 512], akc_channel=None, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer4') external_l5_xyz, external_l5_points = feature_encoding_layer( l4_xyz, l4_points, npoint=8, radius=1.6, sigma=8 * sigma, K=8, mlp=[512, 512, 512], akc_channel=None, is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='external_layer5') external_l6_scene_feature = tf.reduce_mean(external_l5_points, axis=1, keepdims=True) external_scene_feature = tf_util.dropout(external_l6_scene_feature, keep_prob=0.5, is_training=is_training, scope='external_dp') external_scene_feature = tf_util.conv1d(external_scene_feature, num_class, 1, padding='VALID', activation_fn=None, weight_decay=weight_decay, scope='external_fc') # Feature decoding layers l3_points = feature_decoding_layer(l3_xyz, l4_xyz, l3_points, l4_points, 0.8, 8 * sigma, 8, [512, 512], is_training, bn_decay, weight_decay, scope='fa_layer1') l2_points_128 = feature_decoding_layer(l2_xyz_128, l3_xyz, l2_points_128, l3_points, 0.4, 4 * sigma, 8, [256, 256], is_training, bn_decay, weight_decay, scope='fa_layer2_128') l2_points = feature_decoding_layer(l2_xyz, l2_xyz_128, l2_points, l2_points_128, 0.4, 4 * sigma, 8, [256, 256], is_training, bn_decay, weight_decay, scope='fa_layer2') l1_points_512 = feature_decoding_layer(l1_xyz_512, l2_xyz, l1_points_512, l2_points, 0.2, 2 * sigma, 8, [256, 256], is_training, bn_decay, weight_decay, scope='fa_layer3_512') l1_points_1024 = feature_decoding_layer(l1_xyz_1024, l1_xyz_512, l1_points_1024, l1_points_512, 0.2, 2 * sigma, 8, [256, 256], is_training, bn_decay, weight_decay, scope='fa_layer3_1024') l1_points = feature_decoding_layer(l1_xyz, l1_xyz_1024, l1_points, l1_points_1024, 0.2, 2 * sigma, 8, [256, 128], is_training, bn_decay, weight_decay, scope='fa_layer3') l0_points = feature_decoding_layer(l0_xyz, l1_xyz, l0_points, l1_points, 0.1, sigma, 8, [128, 128, 128], is_training, bn_decay, weight_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, weight_decay=weight_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, weight_decay=weight_decay, scope='fc2') return net, end_points, external_scene_feature
def get_instance_seg_v1_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v1 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value net = tf.expand_dims(point_cloud, 2) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3) global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand]) net = tf_util.conv2d(concat_feat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5) logits = tf_util.conv2d(net, 2, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv10') logits = tf.squeeze(logits, [2]) # BxNxC return logits, end_points
def get_model(point_cloud, is_training, POINT_DIM, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) # CONV net = tf_util.conv2d(input_image, 64, [1, POINT_DIM], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) points_feat1 = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # MAX pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point, 1], padding='VALID', scope='maxpool1') # FC pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1]) pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) pc_feat1 = tf_util.fully_connected(pc_feat1, 128, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) print(pc_feat1) # CONCAT pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1]) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) # CONV net = tf_util.conv2d(points_feat1_concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6') net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7') net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 13, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv8') net = tf.squeeze(net, [2]) return net
def get_instance_seg_v2_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v2 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 1]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 32, [0.4, 0.8, 1.6], [64, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[128, 256, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2) l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [128, 128], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [128, 128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128, 128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') logits = tf_util.conv1d(net, 2, 1, padding='VALID', activation_fn=None, scope='conv1d-fc2') return logits, end_points
def get_model_rbf0_gan(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net_no_bn(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3) #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids = tf.get_variable('centroids', [1, 1, 3, 1024], initializer=tf.constant_initializer( 0.2 * np.random.randn(1, 1, 3, 1024)), dtype=tf.float32) feature = tf.tile(point_cloud_transformed, [1, 1, 1, 1024]) bias = tf.tile(centroids, [batch_size, 1024, 1, 1]) net = tf.subtract(feature, bias) net = tf.norm(net, axis=2, keep_dims=True) net = tf.exp(-net) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) #net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, # scope='fc0', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') net = tf_util.fully_connected(net, 512, bn=False, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=False, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 41, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_model(sequence, is_training, num_classes=10, bn_decay=0.999, weight_decay=0.0001, sn=4, pool_t=False, pool_first=False, freeze_bn=False): """ SeNot Net, input is BxTxHxWx3, output Bx10 """ bsize = sequence.get_shape()[0].value end_points = {} channel_stride = [(64, 1), (128, 2), (256, 2), (512, 2)] # res block options num_blocks = [2, 2, 2, 2] topks = [None, sn, sn, None] shrink_ratios = [None, 2, None, None] net = tf_util.conv3d(sequence, 64, [1, 3, 3], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv0') net = tf_util.max_pool3d(net, [1, 3, 3], stride=[1, 2, 2], scope='pool0', padding='SAME') for gp, cs in enumerate(channel_stride): n_channels = cs[0] stride = cs[1] with tf.variable_scope('group{}'.format(gp)): for i in range(num_blocks[gp]): with tf.variable_scope('block{}'.format(i)): end_points['res{}_{}_in'.format(gp, i)] = net if i == 0: net_bra = tf_util.conv3d(net, n_channels, [1, 3, 3], stride=[1, stride, stride], bn=True, bn_decay=bn_decay, \ is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1') else: net_bra = tf_util.preact_bn_for_conv3d(net, is_training=is_training, bn_decay=bn_decay, \ scope='preact', freeze_bn=freeze_bn) net_bra = tf_util.conv3d(net_bra, n_channels, [1, 3, 3], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \ is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1') net_bra = tf_util.conv3d(net_bra, n_channels, [1, 3, 3], stride=[1, 1, 1], bn=False, bn_decay=bn_decay, \ is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv2') if net.get_shape()[-1].value != n_channels: net = tf_util.conv3d(net, n_channels, [1, 1, 1], stride=[1, stride, stride], bn=False, bn_decay=bn_decay, \ is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='convshortcut') net = net + net_bra if i == 1: net = tf_util.preact_bn_for_conv3d(net, is_training=is_training, bn_decay=bn_decay, \ scope='bnlast', freeze_bn=freeze_bn) end_points['res{}_{}_mid'.format(gp, i)] = net if topks[gp] is not None: c = net.get_shape()[-1].value net_pointnet, end_point = net_utils.senota_module(net, k=topks[gp], mlp=[c//4,c//2], scope='pointnet', is_training=is_training, bn_decay=bn_decay, \ weight_decay=weight_decay, distance='l2', activation_fn=None, freeze_bn=freeze_bn, shrink_ratio=shrink_ratios[gp]) net += net_pointnet end_points['pointnet{}_{}'.format(gp, i)] = end_point end_points['after_pointnet{}_{}'.format(gp, i)] = net net = tf.nn.relu(net) end_points['res{}_{}_out'.format(gp, i)] = net net = tf.reduce_mean(net, [1, 2, 3]) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp') net = tf_util.fully_connected(net, num_classes, activation_fn=None, weight_decay=weight_decay, scope='fc') return net, end_points
def get_model_elm(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) random_weights = tf.constant(np.random.randn(3, 4096), dtype=tf.float32) random_weights1 = tf.expand_dims(random_weights, 0) random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #2 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #4 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #8 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #16 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #32 net = tf.matmul(point_cloud, random_weights1) net = tf.expand_dims(net, 2) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, scope='fc0', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features
def get_model_rbf_transform(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) c1 = 64 #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids = tf.get_variable('centroids', [1, 1, 3, c1], initializer=tf.constant_initializer( 0.5 * np.random.randn(1, 1, 3, c1)), dtype=tf.float32) net = tf.subtract( tf.tile(tf.expand_dims(point_cloud_transformed, 3), [1, 1, 1, c1]), tf.tile(centroids, [batch_size, num_point, 1, 1])) net = tf.norm(net, axis=2, keep_dims=True) net = tf.exp(-net) with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) c2 = 256 #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids2 = tf.get_variable('centroids2', [1, 1, c2, 64], initializer=tf.constant_initializer( 0.5 * np.random.randn(1, 1, c2, 64)), dtype=tf.float32) net = tf.subtract(tf.tile(net_transformed, [1, 1, c2, 1]), tf.tile(centroids2, [batch_size, num_point, 1, 1])) net = tf.norm(net, axis=3, keep_dims=True) net = tf.exp(-net) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_model_rbf3(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3) #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) c1 = 512 c2 = 8 centroids = tf.get_variable('centroids', [1, 1, 3, c1], initializer=tf.constant_initializer( np.random.randn(1, 1, 3, c1)), dtype=tf.float32) sub_centroids = tf.get_variable('sub_centroids', [1, 1, 3, c2], initializer=tf.constant_initializer( 0.05 * np.random.randn(1, 1, 3, c2)), dtype=tf.float32) #sub_centroids = tf.constant(0.05*np.random.randn(1, 1, 3, c2), dtype=tf.float32) sub_bias = tf.add( tf.tile(tf.expand_dims(sub_centroids, 4), [1, 1, 1, 1, c1]), tf.tile(tf.expand_dims(centroids, 3), [1, 1, 1, c2, 1])) sub_bias = tf.tile(sub_bias, [batch_size, 1024, 1, 1, 1]) sub_feature = tf.tile(tf.expand_dims(point_cloud_transformed, 4), [1, 1, 1, c2, c1]) sub_net = tf.exp(-tf.square( tf.norm( tf.subtract(sub_feature, sub_bias), ord=3, axis=2, keep_dims=True)) ) sub_net = tf.squeeze(sub_net) sub_net = tf.transpose(sub_net, perm=[0, 1, 3, 2]) sub_net = tf_util.max_pool2d(sub_net, [num_point, 1], stride=[1, 1], padding='VALID', scope='maxpool') #sub_net = tf_util.conv2d(sub_net, 16, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='mini_conv1', bn_decay=bn_decay) sub_net = tf_util.conv2d(sub_net, 2, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='mini_conv2', bn_decay=bn_decay) sub_net = tf.squeeze(sub_net) feature = tf.tile(point_cloud_transformed, [1, 1, 1, c1]) bias = tf.tile(centroids, [batch_size, 1024, 1, 1]) net = tf.subtract(feature, bias) #net = tf.exp(net) net = tf.norm(net, ord=3, axis=2, keep_dims=True) net = tf.exp(-tf.square(net)) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.expand_dims(tf.squeeze(net), 2) features = tf.concat([net, sub_net], axis=2) # Symmetric function: max pooling #features = tf_util.max_pool2d(net, [num_point,1], # padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) #net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, # scope='fc0', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_model_rbf(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3) c1 = 1024 #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids = tf.get_variable('centroids', [1, 1, 3, c1], initializer=tf.constant_initializer( 0.5 * np.random.randn(1, 1, 3, c1)), dtype=tf.float32) #the per-centroids weights to change the shape of the multi-norm weights = tf.get_variable( 'weights', [1, 1, 4, c1], initializer=tf.constant_initializer(0.01 * np.random.randn(1, 1, 3, c1)), ) feature = tf.tile(point_cloud_transformed, [1, 1, 1, c1]) bias = tf.tile(centroids, [batch_size, num_point, 1, 1]) net = tf.subtract(feature, bias) net = tf.exp(net) net = tf.exp(-tf.concat( [ tf.norm(net, ord=0.5, axis=2, keep_dims=True), #tf.norm(net, ord=0.8, axis=2, keep_dims=True), tf.norm(net, ord=1, axis=2, keep_dims=True), #tf.norm(net, ord=1.5, axis=2, keep_dims=True), tf.norm(net, ord=2, axis=2, keep_dims=True), #tf.norm(net, ord=3, axis=2, keep_dims=True), #tf.norm(net, ord=4, axis=2, keep_dims=True), tf.norm(net, ord=np.inf, axis=2, keep_dims=True), ], axis=2)) net = tf.multiply(net, tf.tile(weights, [batch_size, num_point, 1, 1])) #net = tf.exp(-net) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.transpose(features, perm=[0, 1, 3, 2]) net = tf_util.conv2d(net, 3, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='mini_conv1', bn_decay=bn_decay) net = tf.reshape(net, [batch_size, -1]) #net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, # scope='fc0', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_model(point_cloud, is_training, bn_decay=None): """ Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """ ############# PointNet++ 分割部分,先采样,再分组,并分层提取出点云特征###################### batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) # Set abstraction layers, 分层特征提取(加SSG策略,single scale grouping) l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers, 对特征进行插值,恢复到之前的点数,最后对每个点进行分类 l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') ##(B,L,50) return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} k = 20 print(point_cloud) adj_matrix = tf_util.pairwise_distance(point_cloud) print(adj_matrix) nn_idx = tf_util.knn(adj_matrix, k=k) print(nn_idx) edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k) with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(edge_feature, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) adj_matrix = tf_util.pairwise_distance(point_cloud_transformed) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(point_cloud_transformed, nn_idx=nn_idx, k=k) net = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn1', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net1 = net adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k) net = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn2', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net2 = net adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k) net = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn3', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net3 = net adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k) net = tf_util.conv2d(edge_feature, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn4', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net4 = net net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='agg', bn_decay=bn_decay) net = tf.reduce_max(net, axis=1, keep_dims=True) # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx9 gray image """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) k = 20 weight_decay = 0 # 添加的 adj = tf_util.pairwise_distance(point_cloud[:, :, 6:]) # in:B*N*3 out:B*N*N nn_idx = tf_util.knn(adj, k=k) # (batch, num_points, k) # out:B*N*K edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k) # out:B*N*K*18 out1 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv1', bn_decay=bn_decay, is_dist=True) out2 = tf_util.conv2d(out1, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv2', bn_decay=bn_decay, is_dist=True) net_1 = tf.reduce_max(out2, axis=-2, keep_dims=True) adj = tf_util.pairwise_distance(net_1) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k) out3 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv3', bn_decay=bn_decay, is_dist=True) out4 = tf_util.conv2d(out3, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv4', bn_decay=bn_decay, is_dist=True) net_2 = tf.reduce_max(out4, axis=-2, keep_dims=True) adj = tf_util.pairwise_distance(net_2) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k) out5 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv5', bn_decay=bn_decay, is_dist=True) # out6 = tf_util.conv2d(out5, 64, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, weight_decay=weight_decay, # scope='adj_conv6', bn_decay=bn_decay, is_dist=True) net_3 = tf.reduce_max(out5, axis=-2, keep_dims=True) out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='adj_conv7', bn_decay=bn_decay, is_dist=True) out_max = tf_util.max_pool2d(out7, [num_point, 1], padding='VALID', scope='maxpool') expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3]) # CONV net = tf_util.conv2d(concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='seg/conv1', is_dist=True) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='seg/conv2', is_dist=True) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 13, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='seg/conv3', is_dist=True) net = tf.squeeze(net, [2]) return net
def get_model(point_input, labels_pl, cls_label, is_training, bn_decay=None, cls_flag=0, coarse_flag=0, mmd_flag=0, pfs_flag=False, fully_concate=False): """ Seg, input is BxNx6, output BxNx50 """ batch_size = point_input.get_shape()[0].value num_point1 = point_input.get_shape()[1].value num_point2 = int(np.floor(num_point1 / 4.0)) num_point3 = int(np.floor(num_point2 / 4.0)) # num_point4 = int(np.floor(num_point3 / 4.0)) end_points = {} k = 10 pk = 10 de1_1 = 1 de1_2 = 2 de2_1 = 1 de2_2 = 2 de3_1 = 1 de3_2 = 2 cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point1, 1]) point_input1 = point_input point_cloud1 = tf.slice(point_input1, [0, 0, 0], [-1, -1, 3]) r1_11 = 0 r1_12 = 0.1 r1_21 = 0 r1_22 = 0.2 r2_11 = 0 r2_12 = 0.4 r2_21 = 0 r2_22 = 0.8 r3_11 = 0 r3_12 = 1.6 r3_21 = 0 r3_22 = 3.2 p1_1 = 0 p1_2 = 0.4 p2_1 = 0 p2_2 = 1.6 # activation_fn = tf.math.softplus activation_fn = tf.nn.relu ################################################################################################## # Hierarchy 1 point_input = tf.concat([point_input, cls_label_one_hot], axis=-1) hie_matrix1 = tf.math.maximum( tf.sqrt(tf_util.pairwise_distance(point_cloud1)), 1e-20) # dist_matrix1_1 = hie_matrix1 # dist_matrix1_2 = hie_matrix1 dist_matrix1_1 = None dist_matrix1_2 = None adj_matrix = tf_util.pairwise_distance(point_cloud1) net1_1 = pan_util.point_atrous_conv(point_input, adj_matrix, dist_matrix1_1, k, de1_1, r1_11, r1_12, 64, scope='page1_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) adj_matrix = tf_util.pairwise_distance(net1_1) net1_2 = pan_util.point_atrous_conv(net1_1, adj_matrix, dist_matrix1_2, k, de1_2, r1_21, r1_22, 64, scope='page1_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) net = tf.squeeze(net1_2) ################################################################################################## # Hierarchy 2 dist_threshold1 = False net, p1_idx, _, point_cloud2 = pan_util.edge_preserve_sampling( net, point_cloud1, num_point2, hie_matrix1, dist_threshold1, pk, 1, p1_1, p1_2, pfs_flag, atrous_flag=False) # point_cloud2 = gather_point(point_cloud1, p1_idx) hie_matrix2 = tf.math.maximum( tf.sqrt(tf_util.pairwise_distance(point_cloud2)), 1e-20) # dist_matrix2_1 = hie_matrix2 # dist_matrix2_2 = hie_matrix2 dist_matrix2_1 = None dist_matrix2_2 = None adj_matrix = tf_util.pairwise_distance(net) net2_1 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix2_1, k, de2_1, r2_11, r2_12, 128, scope='page2_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) adj_matrix = tf_util.pairwise_distance(net2_1) net2_2 = pan_util.point_atrous_conv(net2_1, adj_matrix, dist_matrix2_2, k, de2_2, r2_21, r2_22, 128, scope='page2_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) net = tf.squeeze(net2_2) ################################################################################################## # Hierarchy 3 dist_threshold2 = False net, p2_idx, _, point_cloud3 = pan_util.edge_preserve_sampling( net, point_cloud2, num_point3, hie_matrix2, dist_threshold2, pk, 1, p2_1, p2_2, pfs_flag, atrous_flag=False) # point_cloud3 = gather_point(point_cloud2, p2_idx) hie_matrix3 = tf.math.maximum( tf.sqrt(tf_util.pairwise_distance(point_cloud3)), 1e-20) # dist_matrix3_1 = hie_matrix3 # dist_matrix3_2 = hie_matrix3 dist_matrix3_1 = None dist_matrix3_2 = None adj_matrix = tf_util.pairwise_distance(net) net3_1 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix3_1, k, de3_1, r3_11, r3_12, 256, scope='page3_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) adj_matrix = tf_util.pairwise_distance(net3_1) net3_2 = pan_util.point_atrous_conv(net3_1, adj_matrix, dist_matrix3_2, k, de3_2, r3_21, r3_22, 256, scope='page3_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) ################################################################################################## # Embeded Features net = tf_util.conv2d(net3_2, 1024, [1, 1], padding='VALID', stride=[1, 1], activation_fn=activation_fn, bn=True, is_training=is_training, scope='encoder', bn_decay=bn_decay) net = tf.reduce_max(net, axis=1, keepdims=True) net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, activation_fn=None, scope='rg1', bn_decay=bn_decay) if mmd_flag > 0: end_points['embedding'] = net else: end_points['embedding'] = None net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='rgdp1') if cls_flag > 0: cls_net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, activation_fn=activation_fn, scope='cls_rg', bn_decay=bn_decay) cls_net = tf_util.dropout(cls_net, keep_prob=0.5, is_training=is_training, scope='cls_rgdp') cls_net = tf_util.fully_connected(cls_net, NUM_CATEGORIES, activation_fn=None, scope='cls_fc') end_points['cls_pred'] = cls_net else: end_points['cls_pred'] = None global_feature_size = 1024 net = tf_util.fully_connected(net, global_feature_size, bn=True, is_training=is_training, activation_fn=activation_fn, scope='rg2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='rgdp2') net = tf.reshape(net, [batch_size, 1, 1, global_feature_size]) net = tf.tile(net, [1, num_point3, 1, 1]) net = tf.concat([net, net3_2], axis=-1) net = tf_util.conv2d(net, 512, [1, 1], padding='VALID', stride=[1, 1], activation_fn=activation_fn, bn=True, is_training=is_training, scope='decoder', bn_decay=bn_decay) if coarse_flag > 0: coarse_net = tf.squeeze(net) coarse_net = tf_util.conv1d(coarse_net, 128, 1, padding='VALID', bn=True, activation_fn=activation_fn, is_training=is_training, scope='coarse_fc1', bn_decay=bn_decay) coarse_net = tf_util.dropout(coarse_net, keep_prob=0.5, is_training=is_training, scope='cdp1') coarse_net = tf_util.conv1d(coarse_net, 50, 1, padding='VALID', activation_fn=None, scope='coarse_fc2') coarse_labels_pl = tf_util.gather_labels(labels_pl, p1_idx) coarse_labels_pl = tf_util.gather_labels(coarse_labels_pl, p2_idx) end_points['coarse_pred'] = coarse_net end_points['coarse_label'] = coarse_labels_pl else: end_points['coarse_pred'] = None end_points['coarse_label'] = None ################################################################################################## # Hierarchy 3 adj_matrix = tf_util.pairwise_distance(net) net3_2 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix3_2, k, de3_2, r3_21, r3_22, 256, scope='pagd3_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) net = tf.concat([net3_2, net3_1], axis=-1) if fully_concate: net3_2 = tf.squeeze(net3_2) adj_matrix = tf_util.pairwise_distance(net) net3_1 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix3_1, k, de3_1, r3_11, r3_12, 256, scope='pagd3_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) net3_1 = tf.squeeze(net3_1) ################################################################################################## # Hierarchy 2 idx, weight = pan_util.three_nn_upsampling(point_cloud2, point_cloud3) net3_1 = three_interpolate(net3_1, idx, weight) if fully_concate: net3_2 = three_interpolate(net3_2, idx, weight) net = tf.concat([tf.expand_dims(net3_1, 2), net2_2], axis=-1) adj_matrix = tf_util.pairwise_distance(net) net2_2 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix2_2, k, de2_2, r2_21, r2_22, 128, scope='pagd2_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) net = tf.concat([net2_2, net2_1], axis=-1) if fully_concate: net2_2 = tf.squeeze(net2_2) adj_matrix = tf_util.pairwise_distance(net) net2_1 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix2_1, k, de2_1, r2_11, r2_12, 128, scope='pagd2_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) net2_1 = tf.squeeze(net2_1) ################################################################################################## # Hierarchy 1 idx, weight = pan_util.three_nn_upsampling(point_cloud1, point_cloud2) net2_1 = three_interpolate(net2_1, idx, weight) net3_1 = three_interpolate(net3_1, idx, weight) if fully_concate: net2_2 = three_interpolate(net2_2, idx, weight) net3_2 = three_interpolate(net3_2, idx, weight) net = tf.concat([tf.expand_dims(net2_1, 2), net1_2], axis=-1) adj_matrix = tf_util.pairwise_distance(net) net1_2 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix1_2, k, de1_2, r1_21, r1_22, 64, scope='pagd1_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) net = tf.concat([net1_2, net1_1], axis=-1) adj_matrix = tf_util.pairwise_distance(net) net1_1 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix1_1, k, de1_1, r1_11, r1_12, 64, scope='pagd1_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn) ################################################################################################## # Final Prediction if fully_concate: net = tf.concat([ net1_1, net1_2, tf.expand_dims(net2_1, 2), tf.expand_dims(net2_2, 2), tf.expand_dims(net3_1, 2), tf.expand_dims(net3_2, 2) ], axis=-1) else: net = tf.concat( [net1_1, tf.expand_dims(net2_1, 2), tf.expand_dims(net3_1, 2)], axis=-1) net = tf.squeeze(net) net = tf.concat([net, cls_label_one_hot], axis=-1) net = tf_util.conv1d(net, 128, 1, padding='VALID', bn=True, activation_fn=activation_fn, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloudz l0_points = None end_points['l0_xyz'] = l0_xyz # Layer 1 l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32, 32, 64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256, 256, 512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Set abstraction layers # Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4). # So we only use NCHW for layer 1 until this issue can be resolved. l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=32, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is B x N x 5, output B x num_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 2]) end_points['l0_xyz'] = l0_xyz # c0 c0_l0_xyz, c0_l0_points, c0_l0_indices = pointSIFT_res_module( l0_xyz, l0_points, radius=0.1, out_channel=64, is_training=is_training, bn_decay=bn_decay, scope='layer0_c0', merge='concat') l1_xyz, l1_points, l1_indices = pointnet_sa_module(c0_l0_xyz, c0_l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') # c1 c0_l1_xyz, c0_l1_points, c0_l1_indices = pointSIFT_res_module( l1_xyz, l1_points, radius=0.25, out_channel=128, is_training=is_training, bn_decay=bn_decay, scope='layer1_c0') l2_xyz, l2_points, l2_indices = pointnet_sa_module(c0_l1_xyz, c0_l1_points, npoint=256, radius=0.2, nsample=32, mlp=[128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') # c2 c0_l2_xyz, c0_l2_points, c0_l2_indices = pointSIFT_res_module( l2_xyz, l2_points, radius=0.5, out_channel=256, is_training=is_training, bn_decay=bn_decay, scope='layer2_c0') c1_l2_xyz, c1_l2_points, c1_l2_indices = pointSIFT_res_module( c0_l2_xyz, c0_l2_points, radius=0.5, out_channel=512, is_training=is_training, bn_decay=bn_decay, scope='layer2_c1', same_dim=True) l2_cat_points = tf.concat([c0_l2_points, c1_l2_points], axis=-1) fc_l2_points = tf_util.conv1d(l2_cat_points, 512, 1, padding='VALID', bn=True, is_training=is_training, scope='conv_2_fc', bn_decay=bn_decay) # c3 l3_xyz, l3_points, l3_indices = pointnet_sa_module(c1_l2_xyz, fc_l2_points, npoint=64, radius=0.4, nsample=32, mlp=[512, 512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512, 512], is_training, bn_decay, scope='fa_layer2') _, l2_points_1, _ = pointSIFT_module(l2_xyz, l2_points, radius=0.5, out_channel=512, is_training=is_training, bn_decay=bn_decay, scope='fa_layer2_c0') _, l2_points_2, _ = pointSIFT_module(l2_xyz, l2_points, radius=0.5, out_channel=512, is_training=is_training, bn_decay=bn_decay, scope='fa_layer2_c1') _, l2_points_3, _ = pointSIFT_module(l2_xyz, l2_points, radius=0.5, out_channel=512, is_training=is_training, bn_decay=bn_decay, scope='fa_layer2_c2') l2_points = tf.concat([l2_points_1, l2_points_2, l2_points_3], axis=-1) l2_points = tf_util.conv1d(l2_points, 512, 1, padding='VALID', bn=True, is_training=is_training, scope='fa_2_fc', bn_decay=bn_decay) l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 256], is_training, bn_decay, scope='fa_layer3') _, l1_points_1, _ = pointSIFT_module(l1_xyz, l1_points, radius=0.25, out_channel=256, is_training=is_training, bn_decay=bn_decay, scope='fa_layer3_c0') _, l1_points_2, _ = pointSIFT_module(l1_xyz, l1_points_1, radius=0.25, out_channel=256, is_training=is_training, bn_decay=bn_decay, scope='fa_layer3_c1') l1_points = tf.concat([l1_points_1, l1_points_2], axis=-1) l1_points = tf_util.conv1d(l1_points, 256, 1, padding='VALID', bn=True, is_training=is_training, scope='fa_1_fc', bn_decay=bn_decay) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer4') _, l0_points, _ = pointSIFT_module(l0_xyz, l0_points, radius=0.1, out_channel=128, is_training=is_training, bn_decay=bn_decay, scope='fa_layer4_c0') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling. tf_util.max_pool2d is DEPRECATED: # net = tf_util.max_pool2d(net, [num_point,1], # padding='VALID', scope='maxpool') net = tf.keras.layers.MaxPooling2D(net, [num_point, 1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_second_model(point_cloud, input_label, is_training, cat_num, part_num, \ batch_size, num_point, weight_decay, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} with tf.variable_scope('transform_net1') as sc: K = 3 transform = get_transform(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) out1 = tf_util.conv2d(input_image, 64, [1, K], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out1_max = tf_util.max_pool2d(out1, [num_point, 1], padding='VALID', scope='maxpool1') out1_max = tf.tile(out1_max, [1, num_point, 1, 1]) out2 = tf_util.conv2d(out1, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out2_max = tf_util.max_pool2d(out2, [num_point, 1], padding='VALID', scope='maxpool2') out2_max = tf.tile(out2_max, [1, num_point, 1, 1]) out3 = tf_util.conv2d(out2, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) out3_max = tf_util.max_pool2d(out3, [num_point, 1], padding='VALID', scope='maxpool3') out3_max = tf.tile(out3_max, [1, num_point, 1, 1]) with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128]) net_transformed = tf.matmul(squeezed_out3, transform) net_transformed = tf.expand_dims(net_transformed, [2]) net_transformed_max = tf_util.max_pool2d(net_transformed, [num_point, 1], padding='VALID', scope='maxpool4') net_transformed_max = tf.tile(net_transformed_max, [1, num_point, 1, 1]) out4 = tf_util.conv2d(net_transformed, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point, 1], padding='VALID', scope='maxpool') # classification network net = tf.reshape(out_max, [batch_size, -1]) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1') net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3') # segmentation network one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) #concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5])#modify by zgh concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out1_max, out4, out5]) net2 = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay) net2 = tf_util.conv2d(net2, part_num, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net, net2, end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None): """ Classification PointNet, input is BxNx4, onehotvec is Bx3, output BxNx2 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1,6], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') print global_feat global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3) print 'Global Feat: ', global_feat global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) print point_feat, global_feat_expand concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand]) print concat_feat net = tf_util.conv2d(concat_feat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5) logits = tf_util.conv2d(net, 2, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv10') logits = tf.squeeze(logits, [2]) # BxNxC print logits print '-----------' #net = tf.concat(axis=3, values=[net, tf.expand_dims(tf.slice(point_cloud, [0,0,0], [-1,-1,3]), 2)]) mask = tf.slice(logits,[0,0,0],[-1,-1,1]) < tf.slice(logits,[0,0,1],[-1,-1,1]) mask = tf.to_float(mask) # BxNx1 mask_count = tf.tile(tf.reduce_sum(mask,axis=1,keep_dims=True), [1,1,3]) # Bx1x3 print mask point_cloud_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) # BxNx3 # ---- Subtract points mean ---- mask_xyz_mean = tf.reduce_sum(tf.tile(mask, [1,1,3])*point_cloud_xyz, axis=1, keep_dims=True) # Bx1x3 mask_xyz_mean = mask_xyz_mean/tf.maximum(mask_count,1) # Bx1x3 point_cloud_xyz_stage1 = point_cloud_xyz - tf.tile(mask_xyz_mean, [1,num_point,1]) print 'Point cloud xyz stage1: ', point_cloud_xyz_stage1 # ---- Regress 1st stage center ---- net = tf.expand_dims(point_cloud_xyz_stage1, 2) print net net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg1-stage1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg2-stage1', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg3-stage1', bn_decay=bn_decay) mask_expand = tf.tile(tf.expand_dims(mask,-1), [1,1,1,256]) masked_net = net*mask_expand print masked_net net = tf_util.max_pool2d(masked_net, [num_point,1], padding='VALID', scope='maxpool-stage1') net = tf.squeeze(net, axis=[1,2]) print net net = tf.concat([net, one_hot_vec], axis=1) net = tf_util.fully_connected(net, 256, scope='fc1-stage1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.fully_connected(net, 128, scope='fc2-stage1', bn=True, is_training=is_training, bn_decay=bn_decay) stage1_center = tf_util.fully_connected(net, 3, activation_fn=None, scope='fc3-stage1') stage1_center = stage1_center + tf.squeeze(mask_xyz_mean, axis=1) # Bx3 end_points['stage1_center'] = stage1_center # ---- Subtract stage1 center ---- point_cloud_xyz_submean = point_cloud_xyz - tf.expand_dims(stage1_center, 1) print 'Point cloud xyz submean: ', point_cloud_xyz_submean net = tf.expand_dims(point_cloud_xyz_submean, 2) print net net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg2', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg3', bn_decay=bn_decay) net = tf_util.conv2d(net, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg4', bn_decay=bn_decay) mask_expand = tf.tile(tf.expand_dims(mask,-1), [1,1,1,512]) masked_net = net*mask_expand print masked_net net = tf_util.max_pool2d(masked_net, [num_point,1], padding='VALID', scope='maxpool2') net = tf.squeeze(net, axis=[1,2]) print net net = tf.concat([net, one_hot_vec], axis=1) net = tf_util.fully_connected(net, 512, scope='fc1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, scope='fc2', bn=True, is_training=is_training, bn_decay=bn_decay) # First 3 are cx,cy,cz, next NUM_HEADING_BIN*2 are for heading # next NUM_SIZE_CLUSTER*4 are for dimension output = tf_util.fully_connected(net, 3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4, activation_fn=None, scope='fc3') print output center = tf.slice(output, [0,0], [-1,3]) center = center + stage1_center # Bx3 end_points['center'] = center heading_scores = tf.slice(output, [0,3], [-1,NUM_HEADING_BIN]) heading_residuals_normalized = tf.slice(output, [0,3+NUM_HEADING_BIN], [-1,NUM_HEADING_BIN]) end_points['heading_scores'] = heading_scores # BxNUM_HEADING_BIN end_points['heading_residuals_normalized'] = heading_residuals_normalized # BxNUM_HEADING_BIN (should be -1 to 1) end_points['heading_residuals'] = heading_residuals_normalized * (np.pi/NUM_HEADING_BIN) # BxNUM_HEADING_BIN size_scores = tf.slice(output, [0,3+NUM_HEADING_BIN*2], [-1,NUM_SIZE_CLUSTER]) # BxNUM_SIZE_CLUSTER size_residuals_normalized = tf.slice(output, [0,3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER], [-1,NUM_SIZE_CLUSTER*3]) size_residuals_normalized = tf.reshape(size_residuals_normalized, [batch_size, NUM_SIZE_CLUSTER, 3]) # BxNUM_SIZE_CLUSTERx3 end_points['size_scores'] = size_scores end_points['size_residuals_normalized'] = size_residuals_normalized end_points['size_residuals'] = size_residuals_normalized * tf.expand_dims(tf.constant(mean_size_arr, dtype=tf.float32), 0) return logits, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Set Abstraction layers l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """ batch_size = point_cloud.shape[0] num_point = point_cloud.shape[1] end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) # Set Abstraction layers l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1') l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer3') end_points['feats_0'] = l0_points # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net # 我觉得这里就算feature map了,但是一个点只用了128个元素来描述了 net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def pointnet_sa_module(cascade_id, xyz, points, bidmap, mlp_configs, block_bottom_center_mm, configs, sgf_config_pls, is_training, bn_decay, scope, bn=True, tnet_spec=None, use_xyz=True, IsShowModel=False): ''' Input cascade_id==0: xyz is grouped_points: (batch_size,nsubblock0,npoint_subblock0,6) points: None bidmap: None Input cascade_id==1: xyz: (batch_size,nsubblock0,3) points: (batch_size,nsubblock0,channel) bidmap: (batch_size,nsubblock1,npoint_subblock1) Medium cascade_id==1: grouped_xyz: (batch_size,nsubblock1,npoint_subblock1,3) new_xyz: (batch_size,nsubblock1,3) group_points: (batch_size,nsubblock1,npoint_subblock1,channel) output cascade_id==1: new_xyz: (batch_size,nsubblock1,3) new_points: (batch_size,nsubblock1,channel) ''' block_bottom_center_mm = tf.cast( block_bottom_center_mm, tf.float32, name='block_bottom_center_mm' ) # gpu_0/sa_layer3/block_bottom_center_mm:0 batch_size = xyz.get_shape()[0].value with tf.variable_scope(scope) as sc: cascade_num = configs['flatten_bm_extract_idx'].shape[ 0] - 1 # include global here (Note: cascade_num does not include global in block_pre_util ) assert configs['sub_block_step_candis'].size == cascade_num - 1 if cascade_id == 0: indrop_keep_mask = tf.get_default_graph().get_tensor_by_name( 'indrop_keep_mask:0') # indrop_keep_mask:0 assert len(xyz.shape) == 3 if bidmap == None: grouped_xyz = tf.expand_dims(xyz, 1) grouped_points = tf.expand_dims(points, 1) new_xyz = None valid_mask = None else: batch_idx = tf.reshape(tf.range(batch_size), [batch_size, 1, 1, 1]) nsubblock = bidmap.get_shape()[1].value npoint_subblock = bidmap.get_shape()[2].value batch_idx_ = tf.tile(batch_idx, [1, nsubblock, npoint_subblock, 1]) bidmap = tf.expand_dims(bidmap, axis=-1, name='bidmap') bidmap_concat = tf.concat( [batch_idx_, bidmap], axis=-1, name='bidmap_concat') # gpu_0/sa_layer0/bidmap_concat:0 # The value for invalid item in bidmap is -17. # On GPU, the responding grouped_xyz and grouped_points is 0. # NOT WORK on CPU !!! # invalid indices comes from merge_blocks_while_fix_bmap # set point_indices_f for invalid points as # NETCONFIG['redundant_points_in_block'] ( shoud be set < -500) valid_mask = tf.greater(bidmap, tf.constant( -500, tf.int32), 'valid_mask') # gpu_0/sa_layer0/valid_mask:0 grouped_xyz = tf.gather_nd( xyz, bidmap_concat, name='grouped_xyz') # gpu_0/sa_layer0/grouped_xyz:0 grouped_points = tf.gather_nd(points, bidmap_concat, name='group_points') if cascade_id == 0 and len(indrop_keep_mask.get_shape()) != 0: grouped_indrop_keep_mask = tf.gather_nd( indrop_keep_mask, bidmap_concat, name='grouped_indrop_keep_mask' ) # gpu_0/sa_layer0/grouped_indrop_keep_mask:0 # new_xyz is the "voxel center" or "mean position of points in the voxel" if configs['mean_grouping_position'] and ( not mlp_configs['block_learning'] == '3DCNN'): new_xyz = tf.reduce_mean(grouped_xyz, -2) else: new_xyz = block_bottom_center_mm[:, :, 3:6] * tf.constant( 0.001, tf.float32) # the mid can be mean or block center, decided by configs['mean_grouping_position'] sub_block_mid = tf.expand_dims( new_xyz, -2, name='sub_block_mid') # gpu_1/sa_layer0/sub_block_mid global_block_mid = tf.reduce_mean(sub_block_mid, 1, keepdims=True, name='global_block_mid') grouped_xyz_submid = grouped_xyz - sub_block_mid grouped_xyz_glomid = grouped_xyz - global_block_mid grouped_xyz_feed = [] if 'raw' in configs['xyz_elements']: grouped_xyz_feed.append(grouped_xyz) if 'sub_mid' in configs['xyz_elements']: grouped_xyz_feed.append(grouped_xyz_submid) if 'global_mid' in configs['xyz_elements']: grouped_xyz_feed.append(grouped_xyz_glomid) grouped_xyz_feed = tf.concat(grouped_xyz_feed, -1) if cascade_id == 0: # xyz must be at the first in feed_data_elements !!!! grouped_points = tf.concat( [grouped_xyz_feed, grouped_points[..., 3:]], -1) if len(indrop_keep_mask.get_shape()) != 0: if InDropMethod == 'set1st': # set all the dropped item as the first item tmp1 = tf.multiply(grouped_points, grouped_indrop_keep_mask) points_1st = grouped_points[:, :, 0:1, :] points_1st = tf.tile(points_1st, [1, 1, grouped_points.shape[2], 1]) indrop_mask_inverse = 1 - grouped_indrop_keep_mask tmp2 = indrop_mask_inverse * points_1st grouped_points = tf.add( tmp1, tmp2, name='grouped_points_droped' ) # gpu_0/sa_layer0/grouped_points_droped #tf.add_to_collection( 'check', grouped_points ) elif InDropMethod == 'set0': valid_mask = tf.logical_and( valid_mask, tf.equal(grouped_indrop_keep_mask, 0), name='valid_mask_droped' ) # gpu_1/sa_layer0/valid_mask_droped elif use_xyz: grouped_points = tf.concat([grouped_xyz_feed, grouped_points], axis=-1) tf.add_to_collection('grouped_xyz', grouped_xyz) tf.add_to_collection('grouped_xyz_submid', grouped_xyz_submid) tf.add_to_collection('grouped_xyz_glomid', grouped_xyz_glomid) if cascade_id > 0 and use_xyz and (not cascade_id == cascade_num - 1): grouped_points = tf.concat([grouped_xyz_feed, grouped_points], axis=-1) nsample = grouped_points.get_shape()[2].value # the conv kernel size if IsShowModel: print( '\n\npointnet_sa_module cascade_id:%d\n xyz:%s\n grouped_xyz:%s\n new_xyz:%s\n grouped_points:%s\n nsample:%d' % (cascade_id, shape_str([xyz]), shape_str([grouped_xyz]), shape_str([new_xyz]), shape_str([grouped_points]), nsample)) new_points = grouped_points if valid_mask != None: new_points = new_points * tf.cast(valid_mask[:, :, :, 0:1], tf.float32) if 'growth_rate' in mlp_configs['point_encoder'][cascade_id]: new_points = tf_util.dense_net( new_points, mlp_configs['point_encoder'][cascade_id], bn, is_training, bn_decay,\ scope = 'dense_cascade_%d_point_encoder'%(cascade_id) , is_show_model = IsShowModel ) else: for i, num_out_channel in enumerate( mlp_configs['point_encoder'][cascade_id]): new_points = tf_util.conv2d(new_points, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv%d' % (i), bn_decay=bn_decay) if configs['Cnn_keep_prob'] < 1: if (not configs['only_last_layer_ineach_cascade'] ) or i == len( mlp_configs['point_encoder'][cascade_id]) - 1: new_points = tf_util.dropout( new_points, keep_prob=configs['Cnn_keep_prob'], is_training=is_training, scope='dropout', name='cnn_dp%d' % (i)) if IsShowModel: print('point encoder1 %d, new_points:%s' % (i, shape_str([new_points]))) if cascade_id == 0: root_point_features = new_points #if InDropMethod == 'set0': # if len(indrop_keep_mask.get_shape()) != 0: # new_points = tf.identity(new_points,'points_before_droped') # gpu_0/sa_layer0/points_before_droped:0 # new_points = tf.multiply( new_points, grouped_indrop_keep_mask, name='droped_points' ) # gpu_0/sa_layer0/droped_points:0 else: root_point_features = None pooling = mlp_configs['block_learning'] if pooling == '3DCNN' and (cascade_id == 0): pooling = 'max' #if pooling=='avg': # new_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1') #elif pooling=='weighted_avg': # with tf.variable_scope('weighted_avg1'): # dists = tf.norm(grouped_xyz,axis=-1,ord=2,keep_dims=True) # exp_dists = tf.exp(-dists * 5) # weights = exp_dists/tf.reduce_sum(exp_dists,axis=2,keep_dims=True) # (batch_size, npoint, nsample, 1) # new_points *= weights # (batch_size, npoint, nsample, mlps_0[-1]) # new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True) if pooling == 'max': # Even the grouped_points and grouped_xyz are 0 for invalid points, the # vaule after mlp will not be. It has to be set as 0 forcely before # pooling. if valid_mask != None: new_points = new_points * tf.cast(valid_mask[:, :, :, 0:1], tf.float32) new_points = tf.identity( new_points, 'points_before_max') # gpu_0/sa_layer0/points_before_max new_points = tf.reduce_max(new_points, axis=[2], keepdims=True, name='points_after_max') #elif pooling=='min': # new_points = tf_util.max_pool2d(-1*new_points, [1,nsample], stride=[1,1], padding='VALID', scope='minpool1') #elif pooling=='max_and_avg': # avg_points = tf_util.max_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='maxpool1') # max_points = tf_util.avg_pool2d(new_points, [1,nsample], stride=[1,1], padding='VALID', scope='avgpool1') # new_points = tf.concat([avg_points, max_points], axis=-1) elif pooling == '3DCNN': new_points = grouped_points_to_voxel_points( cascade_id, new_points, valid_mask, block_bottom_center_mm, configs, grouped_xyz, IsShowVoxelModel=IsShowModel) if IsShowModel: print('voxel points:%s' % (shape_str([new_points]))) for i, num_out_channel in enumerate( mlp_configs['voxel_channels'][cascade_id]): kernel_i = [mlp_configs['voxel_kernels'][cascade_id][i]] * 3 stride_i = [mlp_configs['voxel_strides'][cascade_id][i]] * 3 if new_points.shape[1] % 2 == 0: padding_i = np.array([[0, 0], [1, 0], [1, 0], [1, 0], [ 0, 0 ]]) * mlp_configs['voxel_paddings'][cascade_id][i] else: padding_i = np.array([[0, 0], [1, 1], [1, 1], [1, 1], [ 0, 0 ]]) * mlp_configs['voxel_paddings'][cascade_id][i] new_points = tf.pad(new_points, padding_i, "CONSTANT") if type(num_out_channel) == int: new_points = tf_util.conv3d(new_points, num_out_channel, kernel_i, scope='3dconv_%d' % (i), stride=stride_i, padding='VALID', bn=bn, is_training=is_training, bn_decay=bn_decay, name='points_3dcnn_%d' % (i)) if IsShowModel: print('block learning by 3dcnn %d, new_points:%s' % (i, shape_str([new_points]))) elif num_out_channel == 'max': new_points = tf_util.max_pool3d(new_points, kernel_i, scope='3dmax_%d' % (i), stride=stride_i, padding='VALID') if IsShowModel: print('block learning max pooling %d, new_points:%s' % (i, shape_str([new_points]))) elif num_out_channel == 'avg': new_points = tf_util.avg_pool3d(new_points, kernel_i, scope='3dmax_%d' % (i), stride=stride_i, padding='VALID') if IsShowModel: print('block learning avg pooling %d, new_points:%s' % (i, shape_str([new_points]))) # gpu_0/sa_layer1/3dconv_0/points_3dcnn_0:0 if configs['Cnn_keep_prob'] < 1: if (not configs['only_last_layer_ineach_cascade'] ) or i == len( mlp_configs['voxel_channels'][cascade_id]) - 1: new_points = tf_util.dropout( new_points, keep_prob=configs['Cnn_keep_prob'], is_training=is_training, scope='dropout', name='3dcnn_dp%d' % (i)) # gpu_0/sa_layer4/3dconv_0/points_3dcnn_0:0 new_points = tf.squeeze(new_points, [1, 2, 3]) new_points = tf.reshape( new_points, [batch_size, -1, 1, new_points.shape[-1].value]) if IsShowModel: print('after %s, new_points:%s' % (pooling, shape_str([new_points]))) if 'growth_rate' in mlp_configs['block_encoder'][cascade_id]: new_points = tf_util.dense_net( new_points, mlp_configs['block_encoder'][cascade_id], bn, is_training, bn_decay, scope='dense_cascade_%d_block_encoder' % (cascade_id), is_show_model=IsShowModel) else: for i, num_out_channel in enumerate( mlp_configs['block_encoder'][cascade_id]): new_points = tf_util.conv2d(new_points, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv_post_%d' % (i), bn_decay=bn_decay) if configs['Cnn_keep_prob'] < 1: if (not configs['only_last_layer_ineach_cascade'] ) or i == len( mlp_configs['block_encoder'][cascade_id]) - 1: new_points = tf_util.dropout( new_points, keep_prob=configs['Cnn_keep_prob'], is_training=is_training, scope='dropout', name='cnn_dp%d' % (i)) if IsShowModel: print('block encoder %d, new_points:%s' % (i, shape_str([new_points]))) # (2, 512, 1, 64) new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlps_1[-1]) if IsShowModel: print( 'pointnet_sa_module return\n new_xyz: %s\n new_points:%s\n\n' % (shape_str([new_xyz]), shape_str([new_points]))) #import pdb;pdb.set_trace() # (2, 512, 64) return new_xyz, new_points, root_point_features
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is BxNx5, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 2]) end_points['l0_xyz'] = l0_xyz print(l0_xyz.shape) print(l0_points.shape) # Layer 1 l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 1024, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 256, [0.1, 0.3, 0.5], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, 128, radius=0.6, nsample=64, mlp=[256, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, 16, radius=0.8, nsample=32, mlp=[256, 256, 512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(video, is_training, num_classes=400, bn_decay=0.999, weight_decay=0.0001, pool_t=False, pool_first=False, freeze_bn=False): """ Video Net, input is BxTxHxWx3, output Bx400 """ bsize = video.get_shape()[0].value end_points = {} channel_stride = [(64, 1), (128, 2), (256, 2), (512, 2)] # res block options num_blocks = [3, 4, 23, 3] # pointnet options topks = [None, 4, 4, None] shrink_ratios = [None, 2, None, None] pn = [[0] * 3, [0, 0, 1, 1], [0] * 20 + [1] * 3, [0] * 3] net = tf_util.conv3d( video, 64, [1, 7, 7], stride=[1, 2 if pool_first else 1, 2 if pool_first else 1], bn=True, bn_decay=bn_decay, is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv0') net = tf_util.max_pool3d(net, [1, 3, 3], stride=[1, 2, 2], scope='pool0', padding='SAME') for gp, cs in enumerate(channel_stride): n_channels = cs[0] stride = cs[1] with tf.variable_scope('group{}'.format(gp)): for i in range(num_blocks[gp]): with tf.variable_scope('block{}'.format(i)): end_points['res{}_{}_in'.format(gp, i)] = net if i == 0: net_bra = tf_util.conv3d(net, n_channels, [1, 1, 1], stride=[1, stride, stride], bn=True, bn_decay=bn_decay, \ is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1') else: net_bra = tf_util.conv3d(net, n_channels, [1, 1, 1], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \ is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv1') net_bra = tf_util.conv3d(net_bra, n_channels, [1, 3, 3], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \ is_training=is_training, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv2') net_bra = tf_util.conv3d(net_bra, n_channels * 4, [1, 1, 1], stride=[1, 1, 1], bn=True, bn_decay=bn_decay, \ is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='conv3') if net.get_shape()[-1].value != (n_channels * 4): net = tf_util.conv3d(net, n_channels * 4, [1, 1, 1], stride=[1, stride, stride], bn=True, bn_decay=bn_decay, \ is_training=is_training, activation_fn=None, weight_decay=weight_decay, freeze_bn=freeze_bn, scope='convshortcut') net = net + net_bra end_points['res{}_{}_mid'.format(gp, i)] = net if pn[gp][i]: c = net.get_shape()[-1].value net_pointnet, end_point = net_utils.cp_module(net, k=topks[gp], mlp0=[c//8], mlp=[c//8], scope='pointnet', is_training=is_training, bn_decay=bn_decay, \ weight_decay=weight_decay, distance='l2', activation_fn=None, freeze_bn=freeze_bn, shrink_ratio=shrink_ratios[gp]) net += net_pointnet end_points['pointnet{}_{}'.format(gp, i)] = end_point end_points['after_pointnet{}_{}'.format(gp, i)] = net net = tf.nn.relu(net) end_points['res{}_{}_out'.format(gp, i)] = net net = tf.reduce_mean(net, [1, 2, 3]) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp') net = tf_util.fully_connected(net, num_classes, activation_fn=None, weight_decay=weight_decay, scope='fc') return net, end_points
def get_instance_seg_v1_net_modified(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v1 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value net = tf.expand_dims(point_cloud, 2) c1 = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) c1_mp = tf_util.max_pool2d(c1, [num_point,1], padding='VALID', scope='maxpool1') c1_mp = tf.tile(c1_mp, [1, num_point, 1, 1]) c1 = tf.concat(axis=3, values=[c1, c1_mp]) c2 = tf_util.conv2d(c1, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) c2_mp = tf_util.max_pool2d(c2, [num_point,1], padding='VALID', scope='maxpool2') c2_mp = tf.tile(c2_mp, [1, num_point, 1, 1]) c2 = tf.concat(axis=3, values=[c2, c2_mp]) c3 = tf_util.conv2d(c2, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) c3_mp = tf_util.max_pool2d(c3, [num_point,1], padding='VALID', scope='maxpool3') c3_mp = tf.tile(c3_mp, [1, num_point, 1, 1]) c3 = tf.concat(axis=3, values=[c3, c3_mp]) c4 = tf_util.conv2d(c3, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) c4_mp = tf_util.max_pool2d(c4, [num_point,1], padding='VALID', scope='maxpool4') c4_mp = tf.tile(c4_mp, [1, num_point, 1, 1]) c4 = tf.concat(axis=3, values=[c4, c4_mp]) c5 = tf_util.conv2d(c4, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(c5, [num_point,1], padding='VALID', scope='maxpool') global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3) global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) # concat_feat = tf.concat(axis=3, values=[c3, global_feat_expand]) global_feat_expand = tf.concat(axis=3, values=[c5, global_feat_expand]) d1 = tf_util.conv2d(global_feat_expand, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) d1 = tf.concat(axis=3, values=[d1, c4]) d2 = tf_util.conv2d(d1, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) d2 = tf.concat(axis=3, values=[d2, c3]) d3 = tf_util.conv2d(d2, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) d3 = tf.concat(axis=3, values=[d3, c2]) d4 = tf_util.conv2d(d3, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) d4 = tf.concat(axis=3, values=[d4, c1]) net = tf_util.dropout(d4, is_training, 'dp1', keep_prob=0.5) logits = tf_util.conv2d(net, 2, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv10') logits = tf.squeeze(logits, [2]) # BxNxC return logits, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \ batch_size, num_point, weight_decay, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} with tf.variable_scope('transform_net1') as sc: K = 3 transform = get_transform(point_cloud, is_training, bn_decay, K = 3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) out1 = tf_util.conv2d(input_image, 64, [1,K], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out3 = tf_util.conv2d(out2, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128]) net_transformed = tf.matmul(squeezed_out3, transform) net_transformed = tf.expand_dims(net_transformed, [2]) out4 = tf_util.conv2d(net_transformed, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point,1], padding='VALID', scope='maxpool') # classification network net = tf.reshape(out_max, [batch_size, -1]) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1') net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3') # segmentation network one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5]) net2 = tf_util.conv2d(concat, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay) net2 = tf_util.conv2d(net2, part_num, [1,1], padding='VALID', stride=[1,1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net, net2, end_points
def get_model(point_cloud, is_training, bn_decay=None, output_dim=1): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value # num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') # net = tf_util.fully_connected(net, 1, activation_fn=None, scope='fc3') net = tf_util.fully_connected(net, output_dim, activation_fn=None, scope='fc3') net = tf.squeeze(tf.sigmoid(net)) return net, end_points
def get_model(point_cloud, is_training, num_neighbors, farthest_distance, bn_decay=None): """ Semantic segmentation PointNet, input is BxNx9, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud[:, :, 0:3] l0_points = point_cloud[:, :, 3:9] """Point Enrichment """ new_xyz = l0_xyz # (batch_size, npoint, 3) idx, pts_cnt = query_ball_point(farthest_distance, num_neighbors, l0_xyz, new_xyz) neighbor_xyz = group_point(l0_xyz, idx) neighbor_xyz -= tf.tile(tf.expand_dims(new_xyz, 2), [1, 1, num_neighbors, 1]) neighbor_points = group_point(l0_points, idx) neighbor_representation = tf.concat([neighbor_xyz, neighbor_points], axis=-1) neighbor_representation = tf.reshape(neighbor_representation, (batch_size, num_point, -1)) num_channel = neighbor_representation.get_shape()[2].value points = tf_util.conv1d(point_cloud, num_channel, 1, padding='VALID', bn=True, is_training=is_training, scope='points_fc', bn_decay=bn_decay) neighbor_representation_gp = gating_process( neighbor_representation, num_channel, padding='VALID', is_training=is_training, scope='neighbor_representation_gp', bn_decay=bn_decay) points_gp = gating_process(points, num_channel, padding='VALID', is_training=is_training, scope='points_gp', bn_decay=bn_decay) l0_points = tf.concat([ neighbor_representation_gp * points, points_gp * neighbor_representation ], axis=-1) # Layer 1 l1_xyz, l1_points, l1_indices = pointnet_sa_module_withgab( l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32, 32, 64], mlp2=[64, 64], group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', gab=True) l2_xyz, l2_points, l2_indices = pointnet_sa_module_withgab( l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module_withgab( l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l4_xyz, l4_points, l4_indices = pointnet_sa_module_withgab( l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256, 256, 512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) """Spatial-wise Attention""" input = net output_a = tf_util.conv2d(tf.expand_dims(input, 1), 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv_output_a', bn_decay=bn_decay) output_b = tf_util.conv2d(tf.expand_dims(input, 1), 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv_output_b', bn_decay=bn_decay) output_b = tf.transpose(output_b, [0, 1, 3, 2]) output_a = tf.squeeze(output_a) output_b = tf.squeeze(output_b) energy = tf.matmul(output_a, output_b) attention = tf.nn.softmax(energy, axis=-1) output_d = tf_util.conv2d(tf.expand_dims(input, 1), 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv_output_d', bn_decay=bn_decay) output_d = tf.squeeze(output_d) gamma = tf_util._variable_with_weight_decay('weight_patial', shape=[1], use_xavier=True, stddev=1e-3, wd=0.0) output_SA = tf.matmul(attention, output_d) output_SA = output_SA * gamma + tf.squeeze(input) """Channel-wise Attention""" output_f = tf.transpose(input, [0, 2, 1]) energy = tf.matmul(output_f, input) D = tf.reduce_max(energy, -1) D = tf.expand_dims(D, -1) energy_new = tf.tile(D, multiples=[1, 1, energy.shape[2]]) - energy attention = tf.nn.softmax(energy_new, axis=-1) output_CA = tf.matmul(input, attention) gamma2 = tf_util._variable_with_weight_decay('weightsgamma2m', shape=[1], use_xavier=True, stddev=1e-3, wd=0.0) output_CA = output_CA * gamma2 + input output = output_SA + output_CA end_points['feats'] = output net = tf_util.dropout(output, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 13, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def pointnet_fp_module(cascade_id, num_neighbors, points1, points2, flatten_bidxmap, fbmap_neighbor_idis, mlps_e1, mlps_fp, is_training, bn_decay, scope, configs, bn=True, IsShowModel=False): ''' in Qi's code, 3 larger balls are weighted back-propogated to one point Here, I only back-propogate one Input: points1 (cascade_id=2): (2, 256, 256) points2 (cascade_id=3): (2, 64, 512) flatten_bidxmap: (B,num_point,self.flatbxmap_max_nearest_num,2) [:,:,:,0]: aim_b_index [:,:,:,1]: point_index_in_aimb (useless when cascade_id>0) fbmap_neighbor_idis: (B,num_point,self.flatbxmap_max_nearest_num,1) [:,:,:,2]: index_distance mlps_fp: [256,256] Output: new_points1: (2, 256, 256) ''' if IsShowModel: print( '\n\npointnet_fp_module %s\n points1: %s\n points2: %s\n flatten_bidxmap: %s\n' % (scope, shape_str([points1]), shape_str( [points2]), shape_str([flatten_bidxmap]))) with tf.variable_scope(scope) as sc: assert len(flatten_bidxmap.get_shape()) == 4 if cascade_id == 0: # points1 is grouped point features assert len(points1.get_shape()) == 4 else: # points1 is flat point features assert len(points1.get_shape()) == 3 assert flatten_bidxmap.shape[1] == points1.shape[1] batch_size = points2.get_shape()[0].value batch_idx0 = tf.reshape(tf.range(batch_size), [batch_size, 1, 1, 1]) point1_num = flatten_bidxmap.get_shape()[1].value if cascade_id == 0: # convert grouped points1 to flat point features #flatten_bidxmap_aimbidx1 = flatten_bidxmap[:,:,0,0:2] # (2, 256, 1) #flatten_bidxmap_aimbidx_concat1 = tf.concat( [batch_idx, flatten_bidxmap_aimbidx1], axis=-1 ) #points1 = tf.gather_nd(points1, flatten_bidxmap_aimbidx_concat1 ) num_neighbor0 = num_neighbors[0] disw_theta0 = -1.5 # the abs smaller, more smooth assert num_neighbor0 <= flatten_bidxmap.shape[2].value batch_idx = tf.tile( batch_idx0, [1, point1_num, num_neighbor0, 1]) # (2, 256, 1) flatten_bidxmap_concat1 = tf.concat( [batch_idx, flatten_bidxmap[:, :, 0:num_neighbor0, 0:2]], axis=-1) # [...,[batch_idx,aimb_idx,point_idx_in_aimb] ] points1_nei = tf.gather_nd(points1, flatten_bidxmap_concat1) if num_neighbor0 > 1: dis_weight = tf.nn.softmax( fbmap_neighbor_idis[:, :, 0:num_neighbor0, :] * disw_theta0, axis=2) points1_nei = tf.multiply(points1_nei, dis_weight) points1 = tf.reduce_sum(points1_nei, axis=2) #flatten_bidxmap_aimbidx = flatten_bidxmap[:,:,0,0:1] # (2, 256, 1) #flatten_bidxmap_aimbidx_concat = tf.concat( [batch_idx, flatten_bidxmap_aimbidx],axis=-1 ) # (2, 256, 2) #mapped_points2 = tf.gather_nd(points2, flatten_bidxmap_aimbidx_concat) # (2, 256, 512) # use the inverse distance weighted sum of 3 neighboured point features if cascade_id == 0: num_neighbor = num_neighbors[1] else: num_neighbor = num_neighbors[2] assert num_neighbor <= flatten_bidxmap.shape[2].value neighbor_method = 'A' #----------------------------------- if neighbor_method == 'A': batch_idx = tf.tile(batch_idx0, [1, point1_num, 1, 1]) # (2, 256, 1) # from distance to weight if num_neighbor > 1: disw_theta = -0.5 # the abs smaller, more smooth dis_weight = tf.nn.softmax(fbmap_neighbor_idis * disw_theta, axis=2) for i in range(num_neighbor): flatten_bidxmap_aimbidx_concat_i = tf.concat( [batch_idx, flatten_bidxmap[:, :, i:(i + 1), 0:1]], axis=-1) # (2, 256, 2) mapped_points2_nei_i = tf.gather_nd( points2, flatten_bidxmap_aimbidx_concat_i) # (2, 256, 512) if num_neighbor > 1: mapped_points2_nei_i = tf.multiply( mapped_points2_nei_i, dis_weight[:, :, i:(i + 1), :]) if i == 0: mapped_points2 = mapped_points2_nei_i else: mapped_points2 += mapped_points2_nei_i else: mapped_points2 = mapped_points2_nei_i mapped_points2 = tf.squeeze(mapped_points2, 2) #----------------------------------- if neighbor_method == 'B': batch_idx = tf.tile( batch_idx0, [1, point1_num, num_neighbor, 1]) # (2, 256, 1) flatten_bidxmap_aimbidx_concat = tf.concat( [batch_idx, flatten_bidxmap[:, :, 0:num_neighbor, 0:1]], axis=-1) # (2, 256, 2) mapped_points2_nei = tf.gather_nd( points2, flatten_bidxmap_aimbidx_concat) # (2, 256, 512) if num_neighbor > 1: disw_theta = -0.7 # the abs smaller, more smooth dis_weight = tf.nn.softmax(fbmap_neighbor_idis * disw_theta, axis=2) mapped_points2_nei = tf.multiply(mapped_points2_nei, dis_weight) mapped_points2 = tf.reduce_sum(mapped_points2_nei, 2) #----------------------------------- new_points1 = points1 new_points1 = tf.expand_dims(new_points1, 1) if 'growth_rate' in mlps_e1: new_points1 = tf_util.dense_net(new_points0, mlps_e1, bn, is_training, bn_decay, scope='dense_cascade_%d_fp' % (cascade_id), is_show_model=IsShowModel) else: for i, num_out_channel in enumerate(mlps_e1): new_points1 = tf_util.conv2d(new_points1, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv_encoder1_%d' % (i), bn_decay=bn_decay) if configs['Cnn_keep_prob'] < 1: if (not configs['only_last_layer_ineach_cascade'] ) or i == len(mlps_e1) - 1: new_points1 = tf_util.dropout( new_points1, keep_prob=configs['Cnn_keep_prob'], is_training=is_training, scope='dropout', name='cnn_dp%d' % (i)) if IsShowModel: print('new_points1:%s' % (shape_str([new_points1]))) mapped_points2 = tf.expand_dims(mapped_points2, 1) new_points1 = tf.concat(values=[new_points1, mapped_points2], axis=-1) if IsShowModel: print('after concat new_points1:%s' % (shape_str([new_points1]))) if 'growth_rate' in mlps_fp: new_points1 = tf_util.dense_net(new_points1, mlps_fp, bn, is_training, bn_decay, scope='dense_cascade_%d_fp' % (cascade_id), is_show_model=IsShowModel) else: for i, num_out_channel in enumerate(mlps_fp): new_points1 = tf_util.conv2d(new_points1, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv%d' % (i), bn_decay=bn_decay) if configs['Cnn_keep_prob'] < 1: if (not configs['only_last_layer_ineach_cascade'] ) or i == len(mlps_fp) - 1: new_points1 = tf_util.dropout( new_points1, keep_prob=configs['Cnn_keep_prob'], is_training=is_training, scope='dropout', name='cnn_dp%d' % (i)) if IsShowModel: print('new_points1:%s' % (shape_str([new_points1]))) new_points1 = tf.squeeze(new_points1, [1]) # (2, 256, 256) if IsShowModel: print('new_points1:%s' % (shape_str([new_points1]))) #if IsShowModel: import pdb; pdb.set_trace() return new_points1