def __init__(self, points, features, is_training, setting): bn_decay = setting.get_bn_decay(tf.train.get_global_step()) l0_xyz = points l0_points = None num_class = setting.num_class # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [64, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l3_input_shape, l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [l3_input_shape[0], -1]) net = tf_util.fully_connected(FC1_inputs_shape, net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(FC2_inputs_shape, net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') net = tf_util.fully_connected(FC3_inputs_shape, net, num_class, activation_fn=None, scope='fc3') self.logits = tf.expand_dims(net,axis = 1)
def get_model(point_cloud, cls_label, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2') cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1]) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is BxNx5, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,2]) end_points['l0_xyz'] = l0_xyz # Layer 1 l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 1024, [0.1, 0.4], [16, 128],[[32,64],[64, 128],[64,128]], is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 256, [0.1, 0.5], [32,128],[[64,128], [128,256], [128,256]], is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, 128, radius=0.6, nsample=64, mlp=[256,128,256],mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3') l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, 16, radius=0.8, nsample=32, mlp=[256,256,512],mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, scope='', num_point=None, bn_decay=None, ifglobal=False, bn=True, end_points={}): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [16,32,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1', use_nchw=True, bn = bn) l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2,0.4,0.8], [32,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2', bn = bn) l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3', bn = bn) # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=bn, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['embedding'] = net # net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') # net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) # net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') # net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') end_points['l2_xyz'] = l2_xyz end_points['l3_xyz'] = l3_xyz end_points['l1_xyz'] = l1_xyz end_points['l0_xyz'] = l0_xyz end_points['l2_points'] = l2_points end_points['l3_points'] = l3_points end_points['l1_points'] = l1_points end_points['l0_points'] = l0_points return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value # num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope="layer1", use_nchw=True, ) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope="layer2", ) l3_xyz, l3_points, _ = pointnet_sa_module( l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope="layer3", ) # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected( net, 512, bn=True, is_training=is_training, scope="fc1", bn_decay=bn_decay ) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope="dp1") net = tf_util.fully_connected( net, 256, bn=True, is_training=is_training, scope="fc2", bn_decay=bn_decay ) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope="dp2") net = tf_util.fully_connected(net, 40, activation_fn=None, scope="fc3") return net, end_points
def get_instance_seg_v2_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v2 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 128, [0.2,0.4,0.8], [32,64,128], [[32,32,64], [64,64,128], [64 ,96,128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 32, [0.4,0.8,1.6], [64,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[128,256,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2) l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [128,128], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [128,128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fa_layer3') end_points['feats'] = l0_points # FC layers #print("l0_points",l0_points.shape) #print("l0_points",l0_points) net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay) #print("l0_points",l0_points.shape) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') logits = tf_util.conv1d(net, 2, 1, padding='VALID', activation_fn=None, scope='conv1d-fc2') return logits, end_points
def get_instance_seg_v2_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v2 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 128, [0.2,0.4,0.8], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 32, [0.4,0.8,1.6], [64,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[128,256,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2) l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [128,128], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [128,128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') logits = tf_util.conv1d(net, 2, 1, padding='VALID', activation_fn=None, scope='conv1d-fc2') return logits, end_points
def get_features(point_cloud, out_dim): is_training = tf.constant(True) bn_decay = None """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = tf.shape(point_cloud)[0] num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 4, [0.1, 0.2, 0.4], [4, 4, 4], [[16, 16, 32], [16, 16, 32], [32, 64, 64]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 2, [0.2, 0.4, 0.8], [2, 2, 2], [[8, 8, 16], [16, 16, 32], [16, 32, 32]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[32, 32, 64], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [-1, l3_points.shape[1] * l3_points.shape[2]]) # net = fully_connected(net, 64, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) # net = dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') # net = fully_connected(net, 64, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) # net = dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') net = fully_connected(net, out_dim, activation_fn=None, scope='fc3') return net
def get_model(point_cloud, cls_label, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2') cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1]) # Calculate the relative coordinates generated in the first set abstraction layer # These relative coordinates will be concatenated in the last feature propagation layer dist, idx = three_nn(l0_xyz, l1_xyz) dist = tf.maximum(dist, 1e-10) norm = tf.reduce_sum((1.0/dist),axis=2,keep_dims=True) norm = tf.tile(norm,[1,1,3]) weight = (1.0/dist) / norm l0_virtual_centers = three_interpolate(l1_xyz, idx, weight) l0_xyz_rel = l0_xyz - l0_virtual_centers #l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz_rel, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, batchnorm=False, bn_decay=None, dropout_rate=0.1): """ PointNet++ for TAWSS regression, input is BxNxF, output BxN """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value num_features = point_cloud.get_shape()[2].value l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) # point coordinates if num_features == 3: l0_points = None else: l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1]) # scale information #mid_xyz = {'l0_xyz': l0_xyz} #mid_points = {'l0_points': l0_points} # Set Abstraction layers with multi-scale grouping l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 256, [0.1,0.2,0.4], [16,32,64], [[64,128], [128,128], [128,128]], is_training, bn_decay, dropout_rate, scope='layer1', bn=batchnorm) l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 16, [0.2,0.4,0.8], [16,32,64], [[128],[256],[512]], is_training, bn_decay, dropout_rate, scope='layer2', bn=batchnorm) l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3', bn=batchnorm) #mid_xyz['l2_xyz'] = l2_xyz #mid_points['l2_points'] = l2_points #mid_xyz['l1_xyz'] = l1_xyz #mid_points['l1_points'] = l1_points #mid_xyz['l3_xyz'] = l3_xyz #mid_points['l3_points'] = l3_points # Feature Propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512], is_training, bn_decay, dropout_rate, scope='fp_layer1', bn=batchnorm) l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256], is_training, bn_decay, dropout_rate, scope='fp_layer2', bn=batchnorm) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128], is_training, bn_decay, dropout_rate, scope='fp_layer3', bn=batchnorm) # Fully Connected layers net = tf_util.conv1d(l0_points, 128, 1, scope='fc1', padding='VALID', is_training=is_training, bn=batchnorm, bn_decay=bn_decay) #mid_points['feats'] = net net = tf_util.dropout(net, rate=dropout_rate, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 1, 1, scope='fc2', padding='VALID', activation_fn=None, bn=False) return net#, mid_xyz, mid_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [16,32,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2,0.4,0.8], [32,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_proposal_cls_net(point_cloud, img_seg_map, is_training, bn_decay, end_points): batch_size = point_cloud.shape[0] l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, NUM_CHANNEL - 3]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='cls_layer1') l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 32, [0.4, 0.8, 1.6], [64, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='cls_layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[128, 256, 512], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='cls_layer3') # image feature pooling _img_pixel_size = np.asarray([360, 1200]) box2d_corners, box2d_corners_norm = projection.tf_project_to_image_space( proposal_boxes, calib, _img_pixel_size) # crop and resize ''' # y1, x1, y2, x2 box2d_corners_norm_reorder = tf.stack([ tf.gather(box2d_corners_norm, 1, axis=-1), tf.gather(box2d_corners_norm, 0, axis=-1), tf.gather(box2d_corners_norm, 3, axis=-1), tf.gather(box2d_corners_norm, 2, axis=-1), ], axis=-1) img_rois = tf.image.crop_and_resize( img_seg_map, box2d_corners_norm_reorder, tf.range(0, batch_size), [16,16]) ''' # roi pooling in faster-rcnn img_seg_map = tf_util.conv2d(img_seg_map, 1, 1, padding='VALID', bn=True, is_training=is_training, scope='cls_feature_bottleneck', bn_decay=bn_decay) # feature map index, upper left, bottom right coordinates roi_crops = tf.concat(tf.expand_dims(tf.range(0, batch_size), axis=-1), box2d_corners, axis=-1) img_rois = roi_pooling(img_seg_map, roi_crops, pool_height=16, pool_width=16) img_feats = tf.reshape(img_rois, [batch_size, -1]) # classification point_feats = tf.reshape(l3_points, [batch_size, -1]) # use image only #cls_net = img_feats # use point and image feature cls_net = tf.concat([point_feats, img_feats], axis=1) # use point only #cls_net = point_feats cls_net = tf_util.fully_connected(cls_net, 512, bn=True, is_training=is_training, scope='cls_fc1', bn_decay=bn_decay) cls_net = tf_util.dropout(cls_net, keep_prob=0.5, is_training=is_training, scope='cls_dp1') cls_net = tf_util.fully_connected(cls_net, 256, bn=True, is_training=is_training, scope='cls_fc2', bn_decay=bn_decay) cls_net = tf_util.dropout(cls_net, keep_prob=0.5, is_training=is_training, scope='cls_dp2') cls_net = tf_util.fully_connected(cls_net, NUM_OBJ_CLASSES, activation_fn=None, scope='cls_logits') end_points['cls_logits'] = cls_net return end_points
def get_model(point_cloud, is_training, global_config, bn_decay=None): """ Contact-GraspNet model consisting of a PointNet++ backbone and multiple output heads Arguments: point_cloud {tf.placeholder} -- batch of point clouds is_training {bool} -- train or eval mode global_config {dict} -- config Keyword Arguments: bn_decay {tf.variable} -- batch norm decay (default: {None}) Returns: [dict] -- endpoints of the network """ model_config = global_config['MODEL'] data_config = global_config['DATA'] radius_list_0 = model_config['pointnet_sa_modules_msg'][0]['radius_list'] radius_list_1 = model_config['pointnet_sa_modules_msg'][1]['radius_list'] radius_list_2 = model_config['pointnet_sa_modules_msg'][2]['radius_list'] nsample_list_0 = model_config['pointnet_sa_modules_msg'][0]['nsample_list'] nsample_list_1 = model_config['pointnet_sa_modules_msg'][1]['nsample_list'] nsample_list_2 = model_config['pointnet_sa_modules_msg'][2]['nsample_list'] mlp_list_0 = model_config['pointnet_sa_modules_msg'][0]['mlp_list'] mlp_list_1 = model_config['pointnet_sa_modules_msg'][1]['mlp_list'] mlp_list_2 = model_config['pointnet_sa_modules_msg'][2]['mlp_list'] npoint_0 = model_config['pointnet_sa_modules_msg'][0]['npoint'] npoint_1 = model_config['pointnet_sa_modules_msg'][1]['npoint'] npoint_2 = model_config['pointnet_sa_modules_msg'][2]['npoint'] fp_mlp_0 = model_config['pointnet_fp_modules'][0]['mlp'] fp_mlp_1 = model_config['pointnet_fp_modules'][1]['mlp'] fp_mlp_2 = model_config['pointnet_fp_modules'][2]['mlp'] input_normals = data_config['input_normals'] offset_bins = data_config['labels']['offset_bins'] joint_heads = model_config['joint_heads'] # expensive, rather use random only if 'raw_num_points' in data_config and data_config[ 'raw_num_points'] != data_config['ndataset_points']: point_cloud = gather_point( point_cloud, farthest_point_sample(data_config['ndataset_points'], point_cloud)) end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) if input_normals else None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, npoint_0, radius_list_0, nsample_list_0, mlp_list_0, is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, npoint_1, radius_list_1, nsample_list_1, mlp_list_1, is_training, bn_decay, scope='layer2') if 'asymmetric_model' in model_config and model_config['asymmetric_model']: l3_xyz, l3_points = pointnet_sa_module_msg(l2_xyz, l2_points, npoint_2, radius_list_2, nsample_list_2, mlp_list_2, is_training, bn_decay, scope='layer3') l4_xyz, l4_points, _ = pointnet_sa_module( l3_xyz, l3_points, npoint=None, radius=None, nsample=None, mlp=model_config['pointnet_sa_module']['mlp'], mlp2=None, group_all=model_config['pointnet_sa_module']['group_all'], is_training=is_training, bn_decay=bn_decay, scope='layer4') # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, fp_mlp_0, is_training, bn_decay, scope='fa_layer1') l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, fp_mlp_1, is_training, bn_decay, scope='fa_layer2') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, fp_mlp_2, is_training, bn_decay, scope='fa_layer3') l0_points = l1_points pred_points = l1_xyz else: l3_xyz, l3_points, _ = pointnet_sa_module( l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=model_config['pointnet_sa_module']['mlp'], mlp2=None, group_all=model_config['pointnet_sa_module']['group_all'], is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature Propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, fp_mlp_0, is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, fp_mlp_1, is_training, bn_decay, scope='fa_layer2') l0_points = tf.concat([l0_xyz, l0_points], axis=-1) if input_normals else l0_xyz l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, fp_mlp_2, is_training, bn_decay, scope='fa_layer3') pred_points = l0_xyz if joint_heads: head = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) head = tf_util.dropout(head, keep_prob=0.7, is_training=is_training, scope='dp1') head = tf_util.conv1d(head, 4, 1, padding='VALID', activation_fn=None, scope='fc2') grasp_dir_head = tf.slice(head, [0, 0, 0], [-1, -1, 3]) grasp_dir_head = tf.math.l2_normalize(grasp_dir_head, axis=2) binary_seg_head = tf.slice(head, [0, 0, 3], [-1, -1, 1]) else: # Head for grasp direction grasp_dir_head = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) grasp_dir_head = tf_util.dropout(grasp_dir_head, keep_prob=0.7, is_training=is_training, scope='dp1') grasp_dir_head = tf_util.conv1d(grasp_dir_head, 3, 1, padding='VALID', activation_fn=None, scope='fc3') grasp_dir_head_normed = tf.math.l2_normalize(grasp_dir_head, axis=2) # Head for grasp approach approach_dir_head = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1_app', bn_decay=bn_decay) approach_dir_head = tf_util.dropout(approach_dir_head, keep_prob=0.7, is_training=is_training, scope='dp1_app') approach_dir_head = tf_util.conv1d(approach_dir_head, 3, 1, padding='VALID', activation_fn=None, scope='fc3_app') approach_dir_head_orthog = tf.math.l2_normalize( approach_dir_head - tf.reduce_sum( tf.multiply(grasp_dir_head_normed, approach_dir_head), axis=2, keepdims=True) * grasp_dir_head_normed, axis=2) # Head for grasp width if model_config['dir_vec_length_offset']: grasp_offset_head = tf.norm(grasp_dir_head, axis=2, keepdims=True) elif model_config['bin_offsets']: grasp_offset_head = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1_off', bn_decay=bn_decay) grasp_offset_head = tf_util.conv1d(grasp_offset_head, len(offset_bins) - 1, 1, padding='VALID', activation_fn=None, scope='fc2_off') else: grasp_offset_head = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1_off', bn_decay=bn_decay) grasp_offset_head = tf_util.dropout(grasp_offset_head, keep_prob=0.7, is_training=is_training, scope='dp1_off') grasp_offset_head = tf_util.conv1d(grasp_offset_head, 1, 1, padding='VALID', activation_fn=None, scope='fc2_off') # Head for contact points binary_seg_head = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1_seg', bn_decay=bn_decay) binary_seg_head = tf_util.dropout(binary_seg_head, keep_prob=0.5, is_training=is_training, scope='dp1_seg') binary_seg_head = tf_util.conv1d(binary_seg_head, 1, 1, padding='VALID', activation_fn=None, scope='fc2_seg') end_points['grasp_dir_head'] = grasp_dir_head_normed end_points['binary_seg_head'] = binary_seg_head end_points['binary_seg_pred'] = tf.math.sigmoid(binary_seg_head) end_points['grasp_offset_head'] = grasp_offset_head end_points['grasp_offset_pred'] = tf.math.sigmoid( grasp_offset_head ) if model_config['bin_offsets'] else grasp_offset_head end_points['approach_dir_head'] = approach_dir_head_orthog end_points['pred_points'] = pred_points return end_points
def vlad_forward(xyz, reshaped_input, feature_size=1024, max_samples=4096, cluster_size=64, output_dim=256, gating=True, add_batch_norm=True, is_training=True, bn_decay=None): """Forward pass of a NetVLAD block. Args: reshaped_input: If your input is in that form: 'batch_size' x 'max_samples' x 'feature_size' It should be reshaped in the following form: 'batch_size*max_samples' x 'feature_size' by performing: reshaped_input = tf.reshape(input, [-1, features_size]) Returns: vlad: the pooled vector of size: 'batch_size' x 'output_dim' """ input = tf.reshape(reshaped_input, [-1, max_samples, feature_size]) #msg grouping l1_xyz, l1_points = pointnet_sa_module_msg(xyz, input, 256, [0.1, 0.2, 0.4], [16, 32, 64], [[16, 16, 32], [32, 32, 64], [32, 64, 64]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points, _ = pointnet_sa_module(l1_xyz, l1_points, npoint=None, radius=None, nsample=None, mlp=[256, 512], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') print('l2_points:', l2_points) l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(xyz, l1_xyz, tf.concat([xyz,input],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fa_layer3') print('l0_points shape', l0_points) net = tf_util.conv1d(l0_points, 1, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 1, 1, padding='VALID', activation_fn=None, scope='fc2') m = tf.reshape(net, [-1, 1]) print('m:', m) #constrain weights to [0, 1] m = tf.nn.sigmoid(m) weights = m m = tf.tile(m, [1, cluster_size]) print('m:', m) cluster_weights = tf.get_variable("cluster_weights", [feature_size, cluster_size], initializer=tf.random_normal_initializer( stddev=1 / math.sqrt(feature_size))) activation = tf.matmul(reshaped_input, cluster_weights) # activation = tf.contrib.layers.batch_norm(activation, # center=True, scale=True, # is_training=self.is_training, # scope='cluster_bn') # activation = slim.batch_norm( # activation, # center=True, # scale=True, # is_training=self.is_training, # scope="cluster_bn") if add_batch_norm: activation = slim.batch_norm( activation, center=True, scale=True, is_training=is_training, scope="cluster_bn", fused=False) else: cluster_biases = tf.get_variable("cluster_biases", [cluster_size], initializer=tf.random_normal_initializer( stddev=1 / math.sqrt(feature_size))) activation = activation + cluster_biases activation = tf.nn.softmax(activation) activation_crn = tf.multiply(activation, m) activation = tf.reshape(activation_crn, [-1, max_samples, cluster_size]) a_sum = tf.reduce_sum(activation, -2, keepdims=True) cluster_weights2 = tf.get_variable("cluster_weights2", [1, feature_size, cluster_size], initializer=tf.random_normal_initializer( stddev=1 / math.sqrt(feature_size))) a = tf.multiply(a_sum, cluster_weights2) activation = tf.transpose(activation, perm=[0, 2, 1]) reshaped_input = tf.reshape(reshaped_input, [-1, max_samples, feature_size]) vlad = tf.matmul(activation, reshaped_input) vlad = tf.transpose(vlad, perm=[0, 2, 1]) vlad = tf.subtract(vlad, a) vlad = tf.nn.l2_normalize(vlad, 1) vlad = tf.reshape(vlad, [-1, cluster_size * feature_size]) vlad = tf.nn.l2_normalize(vlad, 1) hidden1_weights = tf.get_variable("hidden1_weights", [cluster_size * feature_size, output_dim], initializer=tf.random_normal_initializer( stddev=1 / math.sqrt(cluster_size))) ##Tried using dropout # vlad=tf.layers.dropout(vlad,rate=0.5,training=self.is_training) vlad = tf.matmul(vlad, hidden1_weights) ##Added a batch norm vlad = tf.contrib.layers.batch_norm(vlad, center=True, scale=True, is_training=is_training, scope='bn') if gating: vlad = context_gating(vlad, add_batch_norm, is_training) return vlad, weights
def get_segmentation_net(self, point_cloud, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v2 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, NUM_CHANNEL - 3]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 4096, [0.1, 0.5], [16, 32], [[16, 16, 32], [32, 32, 64]], is_training, bn_decay, scope='layer1', bn=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 1024, [0.5, 1.0], [16, 32], [[64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer2', bn=True) l3_xyz, l3_points = pointnet_sa_module_msg( l2_xyz, l2_points, 256, [1.0, 2.0], [16, 32], [[128, 196, 256], [128, 196, 256]], is_training, bn_decay, scope='layer3', bn=True) l4_xyz, l4_points = pointnet_sa_module_msg( l3_xyz, l3_points, 64, [2.0, 4.0], [16, 32], [[256, 256, 512], [256, 384, 512]], is_training, bn_decay, scope='layer4', bn=True) # Feature Propagation layers l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [512, 512], is_training, bn_decay, scope='fa_layer2', bn=True) l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512, 512], is_training, bn_decay, scope='fa_layer3', bn=True) l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 256], is_training, bn_decay, scope='fa_layer4', bn=True) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128, 128], is_training, bn_decay, scope='fa_layer5', bn=True) end_points['point_feats'] = tf.concat([l0_xyz, l0_points], axis=-1) # (B, N, 3+C1) end_points['point_feats_fuse'] = tf.concat( [end_points['point_feats'], end_points['point_img_feats']], axis=-1) # (B, N, 3+C1+C2) semantic_features = tf.concat( [l0_points, end_points['point_img_feats']], axis=-1) # (B, N, C1+C2) #end_points['point_feats_fuse'] = end_points['point_feats'] #semantic_features = l0_points # FC layers net = tf_util.dropout(semantic_features, keep_prob=0.5, is_training=is_training, scope='dp0') net = tf_util.conv1d(net, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') logits = tf_util.conv1d(net, NUM_SEG_CLASSES, 1, padding='VALID', activation_fn=None, scope='conv1d-fc2') end_points['foreground_logits'] = logits return end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Set abstraction layers # Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4). # So we only use NCHW for layer 1 until this issue can be resolved. l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 4096, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 1024, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 12800, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 6400, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') attachment_net = tf_util.fully_connected(net, 128 * 40, activation_fn=None, scope='fc3') type_net = tf_util.fully_connected(net, 128 * 4, activation_fn=None, scope='fc4') orientation_net = tf_util.fully_connected(net, 128 * 4, activation_fn=None, scope='fc5') surface_net = tf_util.fully_connected(net, 128 * 4, activation_fn=None, scope='fc6') startstage_net = tf_util.fully_connected(net, 128 * 100, activation_fn=None, scope='fc7') attachment_net = tf.reshape(attachment_net, [batch_size, 128, -1]) type_net = tf.reshape(type_net, [batch_size, 128, -1]) orientation_net = tf.reshape(orientation_net, [batch_size, 128, -1]) surface_net = tf.reshape(surface_net, [batch_size, 128, -1]) startstage_net = tf.reshape(startstage_net, [batch_size, 128, -1]) return attachment_net, type_net, orientation_net, surface_net, startstage_net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) # Apply input-transform network with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3, normals=True) l0_xyz_transformed = tf.matmul(l0_xyz, transform) l0_points_transformed = tf.matmul(l0_points, transform) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz_transformed, l0_points_transformed, 512, [0.1, 0.2, 0.4], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') # Apply feature-transform network after first SA layer with tf.variable_scope('transform_net2') as sc: K = l1_points.get_shape( )[2].value # features were concatenated from multi-scales l1_points_expanded = tf.expand_dims(l1_points, 2) transform = feature_transform_net(l1_points_expanded, is_training, bn_decay, K=K) l1_points_transformed = tf.matmul(l1_points, transform) end_points['transform'] = transform l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points_transformed, 128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer2') cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1]) l0_points = pointnet_fp_module( l0_xyz_transformed, l1_xyz, tf.concat( [cls_label_one_hot, l0_xyz_transformed, l0_points_transformed], axis=-1), l1_points, [128, 128], is_training, bn_decay, scope='fp_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, label, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None ppc = [] l0_xyz, l0_points = pointnet_sa_module_msg(l0_xyz[:,:,0:3], l0_points, 512, radius_list=[0.1,0.2,0.4,0.8], nsample_list=[8,16,32,64], mlp_list=[[32,32,64], [32,32,64], [32,32,64], [32,32,64]], is_training=is_training, bn_decay=bn_decay, knn=True, scope='layer1', use_nchw=False, rearrange=True) l0_xyz, l0_points = pointnet_sa_module_msg(l0_xyz[:,:,0:3], l0_points, 256, radius_list=[0.1,0.2,0.4,0.8], nsample_list=[8,16,32,64], mlp_list=[[64,64,128], [64,64,128], [64,64,128], [64,64,128]], is_training=is_training, bn_decay=bn_decay, knn=True, scope='layer0', use_nchw=False, rearrange=True) l0_points = tf.reshape(l0_points, [batch_size, 512, 256]) print(l0_points.get_shape) with tf.variable_scope('netVLAD_feats'): l2_points = netVLAD(l0_points, 64, is_training=is_training, bn=True, bn_decay=bn_decay) with tf.variable_scope('netVLAD_xyz'): l2_xyz = netVLAD(l0_xyz, 64, is_training=is_training, bn=True, bn_decay=bn_decay) l2_points = tf.concat([l2_points, l2_xyz], axis=-1) l2_points = tf.contrib.layers.fully_connected(inputs=l2_points, num_outputs=256, scope='NetVLAD_fc',biases_initializer=None) l2_points = slim.batch_norm(l2_points, center=True, scale=True, is_training=is_training, scope="cluster_bn", fused=False, decay=bn_decay) ppc = tf.reshape(l2_points, (batch_size, 1, 1024, 16, 1)) ppc = squashing(ppc) ppc = tf.squeeze(ppc) with tf.variable_scope('DigitCaps_Layers_1'): digitCaps = CapsLayer(input_number=1024, output_number=NUM_CLASSES, vec_length=16, out_length=32, layer_type='FC') caps = digitCaps(ppc, is_training=is_training, bn=False) one_hot_label = tf.one_hot(label, depth=NUM_CLASSES, axis=1, dtype=tf.float32) masked_v = tf.matmul(tf.squeeze(caps), tf.reshape(one_hot_label, (-1, NUM_CLASSES, 1)), transpose_a=True) with tf.variable_scope('Reconstruct'): v_j = tf.reshape(masked_v, shape=(batch_size, -1)) fc = tf.contrib.layers.fully_connected(inputs=v_j, num_outputs=128, scope="fc1", biases_initializer=None) fc = slim.batch_norm(fc, center=True, scale=True, is_training=is_training, scope="fc1", fused=False, decay=bn_decay) fc = tf.contrib.layers.fully_connected(inputs=fc, num_outputs=256, scope="fc2", biases_initializer=None) fc = slim.batch_norm(fc, center=True, scale=True, is_training=is_training, scope="fc2", fused=False, decay=bn_decay) fc = tf.contrib.layers.fully_connected(inputs=fc, num_outputs=512, scope="fc3", biases_initializer=None) fc = slim.batch_norm(fc, center=True, scale=True, is_training=is_training, scope="fc3", fused=False, decay=bn_decay) fc = tf.contrib.layers.fully_connected(inputs=fc, num_outputs=1024*3, scope="out", biases_initializer=None, activation_fn=None) net = tf.reshape(fc, (batch_size, 1024, 1, 3)) reconstruct = tf.squeeze(net) end_points['reconstruct'] = tf.reshape(reconstruct, shape=(batch_size, 1024, 3)) return caps, end_points
def get_instance_seg_v2_net(point_cloud, feature_vec, cls_label, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v2 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate feature_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 1]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 32, [0.4, 0.8, 1.6], [64, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[128, 256, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # classification batch_size = tf.shape(feature_vec)[0] cls_net = tf.reshape(l3_points, [batch_size, -1]) cls_net = tf.concat([cls_net, feature_vec], axis=1) cls_net = tf_util.fully_connected(cls_net, 512, bn=True, is_training=is_training, scope='cls_fc1', bn_decay=bn_decay) #cls_net = tf_util.dropout(cls_net, keep_prob=0.4, is_training=is_training, scope='cls_dp1') cls_net = tf_util.fully_connected(cls_net, 256, bn=True, is_training=is_training, scope='cls_fc2', bn_decay=bn_decay) #cls_net = tf_util.dropout(cls_net, keep_prob=0.4, is_training=is_training, scope='cls_dp2') cls_net = tf_util.fully_connected(cls_net, NUM_OBJ_CLASSES, activation_fn=None, scope='cls_logits') end_points['cls_logits'] = cls_net cls_label_pred = tf.argmax(tf.nn.softmax(end_points['cls_logits']), axis=1) end_points['one_hot_vec'] = tf.one_hot(cls_label_pred, NUM_OBJ_CLASSES) end_points['one_hot_gt'] = tf.one_hot(cls_label, NUM_OBJ_CLASSES) # Feature Propagation layers one_hot_vec = tf.cond(is_training, lambda: end_points['one_hot_gt'], lambda: end_points['one_hot_vec']) l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2) l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [128, 128], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [128, 128], is_training, bn_decay, scope='fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128, 128], is_training, bn_decay, scope='fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') logits = tf_util.conv1d(net, 2, 1, padding='VALID', activation_fn=None, scope='conv1d-fc2') return logits, end_points
def trans_pred_net(xyz, flow, scopename, reuse, is_training, bn_decay=None, nfea=12): ######################### # input # xyz: (B x N x 3) # flow: (B x N x 3) # output # pred_trans: (B x N x nfea) ######################### num_point = xyz.get_shape()[1].value with tf.variable_scope(scopename) as myscope: if reuse: myscope.reuse_variables() l0_xyz = xyz l0_points = flow # Set Abstraction layers l1_xyz, l1_points, l1_indices = pointnet_sa_module_msg( l0_xyz, l0_points, 256, [0.1, 0.2], [64, 64], [[64, 64], [64, 64], [64, 128]], is_training, bn_decay, scope='trans_layer1', centralize_points=True) l2_xyz, l2_points, l2_indices = pointnet_sa_module( l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='trans_layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module( l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, use_xyz=True, is_training=is_training, bn_decay=bn_decay, scope='trans_layer3') # Feature Propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='trans_fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='trans_fa_layer2') l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128, 128, 64], is_training, bn_decay, scope='trans_fa_layer3') # FC layers net = tf_util.conv1d(l0_points, 64, 1, padding='VALID', bn=True, is_training=is_training, scope='trans_fc1', bn_decay=bn_decay) net = tf_util.conv1d(net, nfea, 1, padding='VALID', activation_fn=None, scope='trans_fc2') pred_trans = tf.reshape(net, [-1, num_point, nfea]) return pred_trans
def get_model_w_ae_p(self, point_cloud, is_training, bn_decay=None): """" Classification PointNet, input is BxNx3, output Bx40 """ pointnet_util = imp.load_source( 'pointnet_util', os.path.join(os.path.dirname(self.models["test"]), '../utils', "pointnet_util.py")) tf_util = imp.load_source( 'tf_util', os.path.join(os.path.dirname(self.models["test"]), '../utils', "tf_util.py")) from pointnet_util import pointnet_sa_module, pointnet_sa_module_msg batch_size = self.configuration.batch_size num_point = self.configuration.n_input[0] end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) end_points['post_max'] = net net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') end_points['final'] = net net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer2') cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1]) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat( [cls_label_one_hot, l0_xyz, l0_points], axis=-1), l1_points, [128, 128], is_training, bn_decay, scope='fp_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) k = 4 # Set abstraction layers _, tl0_points = pointnet_triple_module(l0_xyz, l0_points, [32, 32], is_training, bn_decay, scope='tlayer1', knn=k) l0_points = tf.concat((l0_points, tl0_points), axis=-1) l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1', use_nchw=True) _, tl1_points = pointnet_triple_module(l1_xyz, l1_points, [64, 64], is_training, bn_decay, scope='tlayer2', knn=k) l1_points = tf.concat((l1_points, tl1_points), axis=-1) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') _, tl2_points = pointnet_triple_module(l2_xyz, l2_points, [128, 128], is_training, bn_decay, scope='tlayer3', knn=k) l2_points = tf.concat((l2_points, tl2_points), axis=-1) l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, n_classes, bn_decay=None, weight_decay=None, inception=True, **kwargs): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} if point_cloud.shape[2] > 3: l0_xyz = point_cloud[:, :, :3] l0_points = point_cloud[:, :, 3:] else: l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz input_image = tf.expand_dims(point_cloud, 2) net, conv_ker = tf_util.inception(input_image, 64, scope='seq_conv1', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 2, 2, 2], return_kernel=True, bn=True, bn_decay=bn_decay, is_training=is_training) end_points['conv_ker'] = conv_ker net, conv_ker = tf_util.inception(input_image, 32, scope='seq_conv2', kernel_heights=[1, 3, 5, 7], kernel_widths=[1, 1, 1, 1], kernels_fraction=[2, 2, 2, 2], return_kernel=True, bn=True, bn_decay=bn_decay, is_training=is_training) conv_net = tf.squeeze(net) l0_points = tf.concat([conv_net, l0_points], axis=-1) # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1', use_nchw=True, weight_decay=weight_decay) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2', weight_decay=weight_decay) l3_xyz, l3_points, _, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3', weight_decay=weight_decay) # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, weight_decay=weight_decay) net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay, weight_decay=weight_decay) net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, n_classes, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3]) farthest_distance = 0.15 num_neighbors = 4 #######Contextual representation new_xyz = l0_xyz # (batch_size, npoint, 3) idx, pts_cnt = query_ball_point(farthest_distance, num_neighbors, l0_xyz, new_xyz) neighbor_xyz = group_point(l0_xyz, idx) neighbor_xyz -= tf.tile(tf.expand_dims(new_xyz, 2), [1, 1, num_neighbors, 1]) neighbor_points = group_point(l0_points, idx) neighbor_representation = tf.concat([neighbor_xyz, neighbor_points], axis=-1) neighbor_representation = tf.reshape(neighbor_representation, (batch_size, num_point, -1)) num_channel = neighbor_representation.get_shape()[2].value points = tf_util.conv1d(point_cloud, num_channel, 1, padding='VALID', bn=True, is_training=is_training, scope='points_fc', bn_decay=bn_decay) neighbor_representation_gp = gating_process( neighbor_representation, num_channel, padding='VALID', is_training=is_training, scope='neighbor_representation_gp', bn_decay=bn_decay) points_gp = gating_process(points, num_channel, padding='VALID', is_training=is_training, scope='points_gp', bn_decay=bn_decay) l0_points_CR = tf.concat([ neighbor_representation_gp * points, points_gp * neighbor_representation ], axis=-1) l0_points = l0_points_CR ########## Positional Representation #num_channel=K idx, pts_cnt = query_ball_point(0.2, 16, l0_xyz, l0_xyz) neighbor_xyz = group_point(l0_xyz, idx) # neighbor_xyz = self.gather_neighbour(xyz, neigh_idx) xyz_tile = tf.tile(tf.expand_dims(l0_xyz, axis=2), [1, 1, tf.shape(idx)[-1], 1]) relative_xyz = xyz_tile - neighbor_xyz #relative_xyz =neighbor_xyz relative_dis = tf.reduce_sum(tf.square(relative_xyz), axis=-1, keepdims=True) encoded_position = tf.concat( [relative_dis, relative_xyz, xyz_tile, neighbor_xyz], axis=-1) encoded_position = tf_util.conv2d(encoded_position, 16, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv011', bn_decay=bn_decay) encoded_neighbours = group_point(l0_points, idx) positional_representation = tf.concat( [encoded_neighbours, encoded_position], axis=-1) positional_representation = tf.reduce_mean(positional_representation, axis=[2], keep_dims=True, name='avgpool') points = tf_util.conv2d(tf.concat( [positional_representation, tf.expand_dims(l0_points_CR, 2)], axis=-1), num_channel * 2, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='attp', bn_decay=bn_decay) points = tf.squeeze(points, [2]) end_points['points'] = points # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [32, 64, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Feature propagation layers l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256, 256], is_training, bn_decay, scope='fa_layer1') l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer2') cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0) cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES]) cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1]) l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat( [cls_label_one_hot, l0_xyz, l0_points], axis=-1), l1_points, [128, 128], is_training, bn_decay, scope='fp_layer3') # FC layers net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) ########## Channel-wise Attention input = net output_a = tf_util.conv2d(tf.expand_dims(input, 1), 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv_output_a', bn_decay=bn_decay) output_b = tf_util.conv2d(tf.expand_dims(input, 1), 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv_output_b', bn_decay=bn_decay) output_b = tf.transpose(output_b, [0, 1, 3, 2]) output_a = tf.squeeze(output_a, [1]) output_b = tf.squeeze(output_b, [1]) energy = tf.matmul(output_b, output_a) D = tf.reduce_max(energy, -1) D = tf.expand_dims(D, -1) energy_new = tf.tile(D, multiples=[1, 1, energy.shape[2]]) - energy attention = tf.nn.softmax(energy_new, axis=-1) output_d = tf_util.conv2d(tf.expand_dims(input, 1), 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv_output_d', bn_decay=bn_decay) output_d = tf.squeeze(output_d, [1]) output_CA = tf.matmul(output_d, attention) gamma2 = tf_util._variable_with_weight_decay('weightsgamma2m', shape=[1], use_xavier=True, stddev=1e-3, wd=0.0) output_CA = output_CA * gamma2 + input ########## Squeeze-and-Excitation ex1 = tf.reduce_mean(input, axis=[1], keep_dims=True, name='avgpool1') print(ex1.get_shape()) ex1 = tf_util.conv1d(ex1, 64, 1, padding='VALID', bn=True, is_training=is_training, scope='ex1', bn_decay=bn_decay) print(ex1.get_shape()) ex1 = tf_util.conv1d(ex1, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='ex2', bn_decay=bn_decay) print(ex1.get_shape()) output2 = input * ex1 output = tf.concat([output_CA, output2], axis=-1) end_points['feats'] = output net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2') return net, end_points