def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1])
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx5, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,2])
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 1024, [0.1, 0.4], [16, 128],[[32,64],[64, 128],[64,128]], is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 256, [0.1, 0.5], [32,128],[[64,128], [128,256], [128,256]], is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, 128, radius=0.6, nsample=64, mlp=[256,128,256],mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, 16, radius=0.8, nsample=32, mlp=[256,256,512],mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_decoder(embedding, is_training, scope='pointnet2_decoder', bn_decay=None, bn=True, end_points = {}):
with tf.name_scope(scope) as sc:
l2_xyz = end_points['l2_xyz']
l3_xyz = end_points['l3_xyz']
l1_xyz = end_points['l1_xyz']
l0_xyz = end_points['l0_xyz']
l2_points = end_points['l2_points']
l3_points = end_points['l3_points']
l1_points = end_points['l1_points']
l0_points = end_points['l0_points']
batch_size = embedding.get_shape()[0].value
# net = tf_util.fully_connected(embedding, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
# net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
# net = tf_util.fully_connected(net, 1024*3, activation_fn=None, scope='fc3')
# pc_fc = tf.reshape(net, (batch_size, -1, 3))
embedding = tf.expand_dims(embedding, axis=1)
l3_points = tf.concat([embedding, l3_points], axis = -1)
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='decoder_fc1', bn_decay=bn_decay)
net = tf_util.conv1d(l0_points, 3, 1, padding='VALID', bn=False, is_training=is_training, scope='decoder_fc2', bn_decay=None, activation_fn=None)
# net = tf_util.conv2d_transpose(net, 3, kernel_size=[1,1], stride=[1,1], padding='VALID', scope='fc2', activation_fn=None)
reconst_pc = tf.reshape(net, [batch_size, -1, 3])
return reconst_pc
def get_model(point_cloud, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None, num_class = NUM_CLASSES):
""" Part segmentation PointNet, input is BxNx3 (XYZ) """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = None
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
###########SEGMENTATION BRANCH
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='seg_fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='seg_dp1')
seg_pred = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='seg_fc2')
return seg_pred
def get_instance_seg_v2_net(point_cloud, one_hot_vec,
is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points,
128, [0.2,0.4,0.8], [32,64,128],
[[32,32,64], [64,64,128], [64 ,96,128]],
is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points,
32, [0.4,0.8,1.6], [64,64,128],
[[64,64,128], [128,128,256], [128,128,256]],
is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points,
npoint=None, radius=None, nsample=None, mlp=[128,256,1024],
mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2)
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points,
[128,128], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points,
[128,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz,
tf.concat([l0_xyz,l0_points],axis=-1), l1_points,
[128,128], is_training, bn_decay, scope='fa_layer3')
end_points['feats'] = l0_points
# FC layers
#print("l0_points",l0_points.shape)
#print("l0_points",l0_points)
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True,
is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay)
#print("l0_points",l0_points.shape)
net = tf_util.dropout(net, keep_prob=0.7,
is_training=is_training, scope='dp1')
logits = tf_util.conv1d(net, 2, 1,
padding='VALID', activation_fn=None, scope='conv1d-fc2')
return logits, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx3 (XYZ) """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = None
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
###########CLASSIFICATION BRANCH
# print(l3_xyz.shape)
# print(l3_points.shape)
net = tf.reshape(l3_points, [batch_size, -1])
# print(net.shape)
# print()
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
# print("Classification feature vector")
class_vector = tf.expand_dims(net, axis=1)
# print(class_vector.shape)
# print()
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2')
class_pred = tf_util.fully_connected(net, NUM_CLASSES, activation_fn=None, scope='fc3')
###########SEGMENTATION BRANCH
# Feature Propagation layers
l3_points_concat = tf.concat([l3_points, class_vector], axis=2)
# l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points_concat, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, class_vector, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
# print(l0_points.shape)
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='seg_fc1', bn_decay=bn_decay)
# print(net.shape)
# print()
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='seg_dp1')
seg_pred = tf_util.conv1d(net, 2, 1, padding='VALID', activation_fn=None, scope='seg_fc2')
# print(seg_pred.shape)
# exit()
# print(class_pred.shape)
# print(seg_pred.shape)
# exit()
return class_pred, seg_pred
def get_model(point_cloud, is_training, num_class, num_embed=5, sigma=0.05, bn_decay=None,is_dist=False):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud[:, :, :3] #[batch_size,number_point,3] contains xyz 3D coordinate information
l0_points = point_cloud[:, :, 3:]#[batch_size,number_point,6] contains other RGBL..
end_points['l0_xyz'] = l0_xyz
# shared encoder
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32, 32, 64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, scope='layer1')
#l1_xyz [batch_size,1024,3] l1_points [batch_size,1024,64] l1_indices [batch_size,1024,32]
l2_xyz, l2_points = pointconv_encoding(l1_xyz, l1_points, npoint=256, radius=0.2, sigma=2 * sigma, K=32, mlp=[ 64, 64, 128], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='layer2')
#l2_xyz [batch_size,256,3] l2_points [batch_size,256,128]
l3_xyz, l3_points = pointconv_encoding(l2_xyz, l2_points, npoint=64, radius=0.4, sigma=4 * sigma, K=32, mlp=[128, 128, 256], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='layer3')
#l3_xyz [batch_size,64,3] l3_points [batch_size,64,256]
l4_xyz, l4_points = pointconv_encoding(l3_xyz, l3_points, npoint=32, radius=0.8, sigma=8 * sigma, K=32, mlp=[256, 256, 512], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='layer4')
#l4_xyz [batch_size,32,3] l4_points [batch_size,32,512]
# semantic decoder
l3_points_sem = pointconv_decoding_depthwise(l3_xyz, l4_xyz, l3_points, l4_points, radius=0.8, sigma=8*sigma, K=16, mlp=[512, 512], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='sem_fa_layer1')
#l3_points_sem = [batch_size,64,512]
l2_points_sem = pointconv_decoding_depthwise(l2_xyz, l3_xyz, l2_points, l3_points_sem, radius=0.4, sigma=4*sigma, K=16, mlp=[256, 256], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='sem_fa_layer2')
# batch_size x256x256
l1_points_sem = pointconv_decoding_depthwise(l1_xyz, l2_xyz, l1_points, l2_points_sem, radius=0.2, sigma=2*sigma, K=16, mlp=[256, 128], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='sem_fa_layer3')
# batch_sizex1024x128
l0_points_sem = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points_sem, [128, 128, 128], is_training, bn_decay, is_dist=is_dist, scope='sem_fa_layer4')
# bx4096x128
# instance decoder
l3_points_ins = pointconv_decoding_depthwise(l3_xyz, l4_xyz, l3_points, l4_points, radius=0.8, sigma=8*sigma, K=16, mlp=[512, 512], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='ins_fa_layer1')
l2_points_ins = pointconv_decoding_depthwise(l2_xyz, l3_xyz, l2_points, l3_points_ins, radius=0.4, sigma=4*sigma, K=16, mlp=[256, 256], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='ins_fa_layer2') # 48x256x256
l1_points_ins = pointconv_decoding_depthwise(l1_xyz, l2_xyz, l1_points, l2_points_ins, radius=0.2, sigma=2*sigma, K=16, mlp=[256, 128], is_training=is_training, bn_decay=bn_decay, is_dist=is_dist, weight_decay=None, scope='ins_fa_layer3') # 48x1024x128
l0_points_ins = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points_ins, [128, 128, 128], is_training, bn_decay, is_dist=is_dist, scope='ins_fa_layer4') # 48x4096x128
# FC layers F_sem
l2_points_sem_up = pointnet_upsample(l0_xyz, l2_xyz, l2_points_sem, scope='sem_up1')#[b,4096,256]
l1_points_sem_up = pointnet_upsample(l0_xyz, l1_xyz, l1_points_sem, scope='sem_up2')#[b,4096,128]
net_sem_0 = tf.add(tf.concat([l0_points_sem, l1_points_sem_up], axis=-1, name='sem_up_concat'), l2_points_sem_up, name='sem_up_add')#[b,4096,256]
net_sem_0 = tf_util.conv1d(net_sem_0, 128, 1, padding='VALID', bn=True, is_training=is_training, is_dist=is_dist, scope='sem_fc1', bn_decay=bn_decay)
#[b,4096,128]
# FC layers F_ins
l2_points_ins_up = pointnet_upsample(l0_xyz, l2_xyz, l2_points_ins, scope='ins_up1')#[b,4096,256]
l1_points_ins_up = pointnet_upsample(l0_xyz, l1_xyz, l1_points_ins, scope='ins_up2')#[b,4096,128]
net_ins_0 = tf.add(tf.concat([l0_points_ins, l1_points_ins_up], axis=-1, name='ins_up_concat'), l2_points_ins_up, name='ins_up_add')#[b,4096,256]
net_ins_0 = tf_util.conv1d(net_ins_0, 128, 1, padding='VALID', bn=True, is_training=is_training, is_dist=is_dist, scope='ins_fc1', bn_decay=bn_decay)
#[b,4096,128]
net_ins_4, net_sem_4 = JSPNet_SIFF_PIFF(net_sem_0,net_ins_0,bn_decay=bn_decay,is_dist=is_dist,is_training=is_training,num_embed=num_embed,num_point=num_point,num_class=num_class)
return net_sem_4,net_ins_4
def get_instance_seg_v2_net(point_cloud, one_hot_vec,
is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points,
128, [0.2,0.4,0.8], [32,64,128],
[[32,32,64], [64,64,128], [64,96,128]],
is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points,
32, [0.4,0.8,1.6], [64,64,128],
[[64,64,128], [128,128,256], [128,128,256]],
is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points,
npoint=None, radius=None, nsample=None, mlp=[128,256,1024],
mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2)
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points,
[128,128], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points,
[128,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz,
tf.concat([l0_xyz,l0_points],axis=-1), l1_points,
[128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True,
is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.7,
is_training=is_training, scope='dp1')
logits = tf_util.conv1d(net, 2, 1,
padding='VALID', activation_fn=None, scope='conv1d-fc2')
return logits, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None, weight_decay=None, feature_channel=0):
""" Semantic segmentation PointNet, input is B x N x3 , output B x num_class """
end_points = {}
num_point = point_cloud.get_shape()[1].value
if feature_channel > 0:
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, feature_channel])
else:
l0_xyz = point_cloud
l0_points = point_cloud
end_points['l0_xyz'] = l0_xyz
num_points = [num_point//8, num_point//32, num_point//128, num_point//256]
_, l0_points = PointASNLSetAbstraction(l0_xyz, l0_points, npoint=num_point, nsample=32, mlp=[16,16,32], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer0', as_neighbor=0, NL=False)
# 1st Res Layer
l1_xyz, l1_1_points = PointASNLSetAbstraction(l0_xyz, l0_points, npoint=num_points[0], nsample=32, mlp=[32,32,64], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer1_1', as_neighbor=8)
_, l1_2_points = PointASNLSetAbstraction(l0_xyz, l0_points, npoint=num_points[0], nsample=32, mlp=[64,64], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer1_2', as_neighbor=0, NL=False)
l1_2_points += l1_1_points
# 2nd Res Layer
l2_xyz, l2_1_points = PointASNLSetAbstraction(l1_xyz, l1_2_points, npoint=num_points[1], nsample=32, mlp=[64,64,128], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay,scope='layer2_1', as_neighbor=4)
_, l2_2_points = PointASNLSetAbstraction(l2_xyz, l2_1_points, npoint=num_points[1], nsample=32, mlp=[128,128], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay,scope='layer2_2', as_neighbor=0, NL=False)
l2_2_points += l2_1_points
# 3rd Res Layer
l3_xyz, l3_1_points = PointASNLSetAbstraction(l2_xyz, l2_2_points, npoint=num_points[2], nsample=32, mlp=[128,128,256], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer3_1', as_neighbor=0)
_, l3_2_points = PointASNLSetAbstraction(l3_xyz, l3_1_points, npoint=num_points[2], nsample=32, mlp=[256,256], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer3_2', as_neighbor=0, NL=False)
l3_2_points += l3_1_points
# 4th Res Layer
l4_xyz, l4_1_points = PointASNLSetAbstraction(l3_xyz, l3_1_points, npoint=num_points[3], nsample=32, mlp=[256,256,512], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer4_1', as_neighbor=0)
_, l4_2_points = PointASNLSetAbstraction(l4_xyz, l4_1_points, npoint=num_points[3], nsample=32, mlp=[512,512], is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay, scope='layer4_2', as_neighbor=0, NL=False)
l4_2_points += l4_1_points
end_points['l1_xyz'] = l1_xyz
# Feature decoding layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_2_points, l4_2_points, [512,512], is_training, bn_decay, scope='fa_layer1', bn=True)
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2', bn=True)
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_2_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3', bn=True)
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4', bn=True)
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', activation_fn=tf.nn.leaky_relu, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, weight_decay=weight_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, weight_decay=weight_decay, scope='fc0')
return net, end_points
def get_model(point_cloud, cls_label, pp_idx, is_training, bn_decay=None):
""" Part segmentation PointNet, input is BxNx6 (XYZ NormalX NormalY NormalZ), output Bx50 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1])
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
orig_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set Abstraction layers
l1_xyz, l1_points, _ = pointnet_sa_module(l0_xyz, orig_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, _ = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
# down sampling to one global point
l3_xyz, l32_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer32')
_, l31_points, _ = pointnet_sa_module(l1_xyz, l1_points, npoint=None, radius=None, nsample=None, mlp=[128,256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer31')
_, l30_points, _ = pointnet_sa_module(l0_xyz, orig_points, npoint=None, radius=None, nsample=None, mlp=[64,128,256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer30')
l3_points = l32_points + l31_points + l30_points
# Feature propagation layers (feature propagation after each SA layer)
fp2_points = pointnet_fp_module(l0_xyz, l3_xyz, tf.concat([cls_label_one_hot, l0_xyz, orig_points],axis=-1), l3_points, [256,128], is_training, bn_decay, scope='dfp_layer1')
fp1_points = pointnet_fp_module(l0_xyz, l2_xyz, tf.concat([cls_label_one_hot, l0_xyz, orig_points],axis=-1), l2_points, [256,128], is_training, bn_decay, scope='dfp_layer2')
fp0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, orig_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='dfp_layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fp_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fp_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot,l0_xyz,orig_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fp_layer3')
# Residual propagation
rfp2_points = pointnet_fp_module(l0_xyz, l2_xyz, tf.concat([cls_label_one_hot, l0_xyz, orig_points],axis=-1), l2_points, [256,128], is_training, bn_decay, scope='rfp_layer1')
rfp1_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, orig_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='rfp_layer2')
rfp_points = rfp2_points + rfp1_points
l0_points = fp2_points + fp1_points + fp0_points + l0_points + rfp_points
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2')
pred = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc3')
if pp_idx is not None:
pp_pred = get_pp_pred(end_points['feats'], pp_idx)
else:
pp_pred = None
return pred, pp_pred, end_points
def _pointnet_fp(self,
arch_dict,
xyz_to,
xyz_from,
feat_to,
feat_from,
is_training,
scope=""):
"""
PointNet Feature Propagation layer (Qi et al. 2017)
:param arch_dict: dictionary describing the architecture of this layer
:param xyz_to: Tensor (batch x num_points x 3). coordinate triplets
:param xyz_from: Tensor (batch x num_points x 3). coordinate triplets
:param feat_to: Tensor (batch x num_points x num_feat). features for each point
:param feat_from: Tensor (batch x num_points x num_feat). features for each point
:param scope: name for the layers
:return: features interpolated to the next layer
"""
li_feats = pointnet_fp_module(xyz_to,
xyz_from,
feat_to,
feat_from,
arch_dict['reverse_mlp'],
is_training,
bn_decay=None,
scope=scope)
return li_feats
def get_displacements(input_points, ske_features, FLAGS, is_training = False, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = FLAGS.batch_size
num_points = FLAGS.point_num_out
point_cloud = input_points
l0_xyz = point_cloud
l0_points = None
# Set Abstraction layers 第一从次2048个点提取1024个点
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1 * FLAGS.radiusScal, nsample=64,
mlp=[64, 64, 128], mlp2=None, group_all=False,
is_training=is_training, bn_decay=bn_decay, scope='layer1') ### 最后一个变量scope相当于变量前缀
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=384, radius=0.2* FLAGS.radiusScal, nsample=64,
mlp=[128, 128, 256], mlp2=None, group_all=False,
is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=128, radius=0.4* FLAGS.radiusScal, nsample=64,
mlp=[256, 256, 512], mlp2=None, group_all=False,
is_training=is_training, bn_decay=bn_decay, scope='layer3')
# PointNet
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=None, radius=None, nsample=None,
mlp=[512, 512, 1024], mlp2=None, group_all=True,
is_training=is_training, bn_decay=bn_decay, scope='layer4')
### Feature Propagation layers ################# featrue maps are interpolated according to coordinate ################
# 根据l4的特征值差值出l3
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [512, 512], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512, 256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256, 128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128, 128, 128], is_training, bn_decay, scope='fa_layer4')
# 加入提取的skeleton特征
# ske_features : batch_size * featrues
ske_features = tf.tile(tf.expand_dims(ske_features, 1), [1, num_points, 1])
l0_points = tf.concat([l0_points, ske_features], axis=-1)
# 特征转变成 displacement
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay )
net = tf_util.conv1d(net, 64, 1, padding='VALID', bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
net = tf_util.conv1d(net, 3, 1, padding='VALID', activation_fn=None, scope='fc3')
displacements = tf.sigmoid(net) * FLAGS.range_max * 2 - FLAGS.range_max
return displacements
def get_model(point_cloud, is_training, bn=True, bn_decay=None):
end_points = {}
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
l0_xyz = point_cloud
l0_points = point_cloud
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=64, mlp=[64,64,128], mlp2=None, group_all=False, \
bn=bn, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128,128,256], mlp2=None, group_all=False, \
bn=bn, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, \
bn=bn, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], \
bn=bn, is_training=is_training, bn_decay=bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], \
bn=bn, is_training=is_training, bn_decay=bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], \
bn=bn, is_training=is_training, bn_decay=bn_decay, scope='fa_layer3')
# semantic segmentation branch
seg_net = l0_points
seg_net = tf_util.conv1d(seg_net,
256,
1,
padding='VALID',
bn=bn,
is_training=is_training,
scope='seg/fc1',
bn_decay=bn_decay)
seg_net = tf_util.conv1d(seg_net,
256,
1,
padding='VALID',
bn=bn,
is_training=is_training,
scope='seg/fc2',
bn_decay=bn_decay)
seg_net = tf.expand_dims(seg_net, axis=2)
print 'PointNet++ Output: ', seg_net
return seg_net
def eva_seg(pcpair,
pred_flow,
nfea=64,
ntransfea=12,
nsmp1=256,
nsmp2=128,
nmask=10):
#####################################################
# Evaluate the motion segmentation from a point cloud
# equipped with a deformation flow.
# input
# pcpair: (B x N x 6)
# output
# pred_trans: (B x N x ntransfea)
# pred_grouping: (B x N x N)
# pred_seg: (B x nmask x N)
# pred_conf: (B x nmask x 1)
#####################################################
num_point = pcpair.get_shape()[1].value
xyz, xyz2 = tf.split(pcpair, [3, 3], axis=2)
pred_trans = trans_pred_net(xyz,
pred_flow,
'TransNet',
False,
tf.constant(False),
None,
nfea=ntransfea)
pred_grouping_sub, fpsidx = grouping_pred_net(xyz,
pred_flow,
pred_trans,
'GroupingNet',
False,
tf.constant(False),
None,
nsmp=nsmp2)
pred_seg_sub, pred_conf = seg_pred_net(xyz,
pred_grouping_sub,
fpsidx,
'SegNet',
False,
tf.constant(False),
None,
nmask=nmask)
pred_conf = tf.nn.sigmoid(pred_conf)
xyz_sub = tf.reshape(tf.gather_nd(xyz, fpsidx), [-1, nsmp2, 3])
#### up sample
pred_grouping = interp_grouping(xyz, pred_grouping_sub, fpsidx, nsmp2,
'InterpNet', False)
pred_seg = tf.transpose(pred_seg_sub, perm=[0, 2, 1]) # B x nsmp x nmask
pred_seg = pointnet_fp_module(xyz,
xyz_sub,
None,
pred_seg, [],
tf.constant(True),
None,
scope='interp_layer_seg')
pred_seg = tf.transpose(pred_seg, perm=[0, 2, 1]) # B x nmask x npoint
return pred_trans, pred_grouping, pred_seg, pred_conf
def build_pointnet2_seg(scope, X, out_dims, is_training, bn_decay):
with tf.variable_scope(scope):
l0_xyz = tf.slice(X, [0,0,0], [-1,-1,3])
l0_points = tf.slice(X, [0,0,3], [-1,-1,0])
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points,
npoint=512, radius=0.2, nsample=64, mlp=[64,64,128],
mlp2=None, group_all=False, is_training=is_training,
bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points,
npoint=128, radius=0.4, nsample=64, mlp=[128,128,256],
mlp2=None, group_all=False, is_training=is_training,
bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points,
npoint=None, radius=None, nsample=None, mlp=[256,512,1024],
mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points,
[256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points,
[256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz,
tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128],
is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True,
is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training,
scope='dp1')
results = []
for idx, out_dim in enumerate(out_dims):
current_result = tf_util.conv1d(net, out_dim, 1, padding='VALID', activation_fn=None, scope='fc2_{}'.format(idx))
results.append(current_result)
return results
def build_pointnet2_seg(X, out_dim, is_training, bn_decay, scope):
n_points = X.get_shape()[1].value
l0_xyz = tf.slice(X, [0,0,0], [-1,-1,3])
l0_points = tf.slice(X, [0,0,3], [-1,-1,0])
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points,
npoint=512, radius=0.2, nsample=64, mlp=[64,64,128],
mlp2=None, group_all=False, is_training=is_training,
bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points,
npoint=128, radius=0.4, nsample=64, mlp=[128,128,256],
mlp2=None, group_all=False, is_training=is_training,
bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points,
npoint=None, radius=None, nsample=None, mlp=[256,512,1024],
mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points,
[256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points,
[256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz,
tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128],
is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True,
is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net, out_dim, 1, padding='VALID', activation_fn=None,
scope='fc2')
return net, 0
def interp_grouping(xyz, pred_grouping, fpsidx, nsmp, scopename, reuse):
""" xyz: B x N x 3,
pred_grouping: B x nsmp x nsmp,
fpsidx: B x nsmp """
num_point = xyz.get_shape()[1].value
with tf.variable_scope(scopename) as myscope:
if reuse:
myscope.reuse_variables()
xyz_sub = tf.reshape(tf.gather_nd(xyz, fpsidx), [-1, nsmp, 3])
# row interp
xyz_aug1 = tf.tile(tf.expand_dims(xyz, 1), (1, nsmp, 1, 1))
xyz_aug1 = tf.reshape(xyz_aug1, (-1, num_point, 3))
xyz_sub_aug1 = tf.tile(tf.expand_dims(xyz_sub, 1), (1, nsmp, 1, 1))
xyz_sub_aug1 = tf.reshape(xyz_sub_aug1, (-1, nsmp, 3))
U_combined = tf.reshape(pred_grouping, (-1, nsmp, 1))
U_combined = pointnet_fp_module(xyz_aug1,
xyz_sub_aug1,
None,
U_combined, [],
tf.constant(True),
None,
scope='interp_layer_row')
U_combined = tf.reshape(U_combined, (-1, nsmp, num_point, 1))
U_combined = tf.transpose(U_combined, perm=(0, 2, 1, 3))
U_combined = tf.reshape(U_combined,
(-1, nsmp, 1)) # B*npoint x nsmp x 1
# column interp
xyz_aug2 = tf.tile(tf.expand_dims(xyz, 1), (1, num_point, 1, 1))
xyz_aug2 = tf.reshape(xyz_aug2, (-1, num_point, 3))
xyz_sub_aug2 = tf.tile(tf.expand_dims(xyz_sub, 1),
(1, num_point, 1, 1))
xyz_sub_aug2 = tf.reshape(xyz_sub_aug2, (-1, nsmp, 3))
U_combined = pointnet_fp_module(xyz_aug2,
xyz_sub_aug2,
None,
U_combined, [],
tf.constant(True),
None,
scope='interp_layer_column')
U_combined = tf.reshape(U_combined, (-1, num_point, num_point))
U_combined = tf.transpose(U_combined, perm=(0, 2, 1))
return U_combined
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1])
# Calculate the relative coordinates generated in the first set abstraction layer
# These relative coordinates will be concatenated in the last feature propagation layer
dist, idx = three_nn(l0_xyz, l1_xyz)
dist = tf.maximum(dist, 1e-10)
norm = tf.reduce_sum((1.0/dist),axis=2,keep_dims=True)
norm = tf.tile(norm,[1,1,3])
weight = (1.0/dist) / norm
l0_virtual_centers = three_interpolate(l1_xyz, idx, weight)
l0_xyz_rel = l0_xyz - l0_virtual_centers
#l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz_rel, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud[:, :, :3]
l0_points = point_cloud[:, :, 3:]
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.1, nsample=32, mlp=[32,32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=256, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=32, mlp=[128,128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.8, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points_sem = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='sem_fa_layer1')
l2_points_sem = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points_sem, [256,256], is_training, bn_decay, scope='sem_fa_layer2')
l1_points_sem = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points_sem, [256,128], is_training, bn_decay, scope='sem_fa_layer3')
l0_points_sem = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points_sem, [128,128,128], is_training, bn_decay, scope='sem_fa_layer4')
# FC layers
net_sem = tf_util.conv1d(l0_points_sem, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='sem_fc1', bn_decay=bn_decay)
net_sem_cache = tf_util.conv1d(net_sem, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='sem_cache', bn_decay=bn_decay)
# ins
l3_points_ins = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='ins_fa_layer1')
l2_points_ins = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points_ins, [256,256], is_training, bn_decay, scope='ins_fa_layer2')
l1_points_ins = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points_ins, [256,128], is_training, bn_decay, scope='ins_fa_layer3')
l0_points_ins = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points_ins, [128,128,128], is_training, bn_decay, scope='ins_fa_layer4')
net_ins = tf_util.conv1d(l0_points_ins, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='ins_fc1', bn_decay=bn_decay)
net_ins = net_ins + net_sem_cache
net_ins = tf_util.dropout(net_ins, keep_prob=0.5, is_training=is_training, scope='ins_dp1')
net_ins = tf_util.conv1d(net_ins, 5, 1, padding='VALID', activation_fn=None, scope='ins_fc4')
k = 40
adj_matrix = tf_util.pairwise_distance_l1(net_ins)
nn_idx = tf_util.knn_thres(adj_matrix, k=k)
nn_idx = tf.stop_gradient(nn_idx)
net_sem = tf_util.get_local_feature(net_sem, nn_idx=nn_idx, k=k)# [b, n, k, c]
net_sem = tf.reduce_max(net_sem, axis=-2, keep_dims=False)
net_sem = tf_util.dropout(net_sem, keep_prob=0.5, is_training=is_training, scope='sem_dp1')
net_sem = tf_util.conv1d(net_sem, num_class, 1, padding='VALID', activation_fn=None, scope='sem_fc4')
return net_sem, net_ins
def get_model(point_cloud, is_training, num_class, bn_decay=None,gripper_feat=None,env_feat=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.05, nsample=32, mlp=[32,64], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.02, nsample=32, mlp=[64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=64, radius=0.04, nsample=32, mlp=[128,256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
#l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, npoint=16, radius=0.08, nsample=32, mlp=[256,256,512], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
#l5_xyz, l5_points, l5_indices = pointnet_sa_module(l4_xyz, l4_points, npoint=4, radius=0.20, nsample=32, mlp=[512,512,1024], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer5')
#if env_feat is None:
# extra_feat = gripper_feat
#else:
# extra_feat = tf.concat([gripper_feat,env_feat],axis=-1)
#extra_feat = tf.expand_dims(extra_feat,axis=1)
#extra_feat = tf.tile(extra_feat,[1,4,1])
#l5_points = tf.concat([l5_points,extra_feat],axis=-1)
# Feature Propagation layers
#l4_points = pointnet_fp_module(l4_xyz, l5_xyz, l4_points, l5_points, [512,256,256], is_training, bn_decay, scope='fa_layer0')
#l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,128,128], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [128,128], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [128,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128,64], is_training, bn_decay, scope='fa_layer4')
#print(l0_points)
# FC layers
net = tf_util.conv1d(l0_points, 64, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
#net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, batchnorm=False, bn_decay=None, dropout_rate=0.1):
"""
PointNet++ for TAWSS regression, input is BxNxF, output BxN
"""
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
num_features = point_cloud.get_shape()[2].value
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) # point coordinates
if num_features == 3: l0_points = None
else: l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1]) # scale information
#mid_xyz = {'l0_xyz': l0_xyz}
#mid_points = {'l0_points': l0_points}
# Set Abstraction layers with multi-scale grouping
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 256, [0.1,0.2,0.4], [16,32,64], [[64,128], [128,128], [128,128]], is_training, bn_decay, dropout_rate, scope='layer1', bn=batchnorm)
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 16, [0.2,0.4,0.8], [16,32,64], [[128],[256],[512]], is_training, bn_decay, dropout_rate, scope='layer2', bn=batchnorm)
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3', bn=batchnorm)
#mid_xyz['l2_xyz'] = l2_xyz
#mid_points['l2_points'] = l2_points
#mid_xyz['l1_xyz'] = l1_xyz
#mid_points['l1_points'] = l1_points
#mid_xyz['l3_xyz'] = l3_xyz
#mid_points['l3_points'] = l3_points
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512], is_training, bn_decay, dropout_rate, scope='fp_layer1', bn=batchnorm)
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256], is_training, bn_decay, dropout_rate, scope='fp_layer2', bn=batchnorm)
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128], is_training, bn_decay, dropout_rate, scope='fp_layer3', bn=batchnorm)
# Fully Connected layers
net = tf_util.conv1d(l0_points, 128, 1, scope='fc1', padding='VALID', is_training=is_training, bn=batchnorm, bn_decay=bn_decay)
#mid_points['feats'] = net
net = tf_util.dropout(net, rate=dropout_rate, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 1, 1, scope='fc2', padding='VALID', activation_fn=None, bn=False)
return net#, mid_xyz, mid_points
def get_model(point_cloud,cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=1024, radius=0.2, nsample=32, mlp=[64,64,128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module_spec(l1_xyz, l1_points, npoint=256, radius=0.4, nsample=32, mlp=[128,256], mlp2=[256], group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2' , knn=True , spec_conv_type = 'mlp', structure='spec' , useloc_covmat = True, pooling='max')
l3_xyz, l3_points, l3_indices = pointnet_sa_module_spec(l2_xyz, l2_points, npoint=64, radius=0.4, nsample=16, mlp=[256,512], mlp2=[512], group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3' , knn=True , spec_conv_type = 'mlp', structure='spec' , useloc_covmat = True, pooling='hier_cluster_pool', csize = 2 )
l4_xyz, l4_points, l4_indices = pointnet_sa_module_spec(l3_xyz, l3_points, npoint=None, radius=None, nsample=None, mlp=[512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer4', knn=True , spec_conv_type = 'mlp', structure='spec' , useloc_covmat = True, pooling='max')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [1024,512], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1])
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
# num_point = 10000
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = None
sampled_points = 256
s1_points = 16
s2_points = 32
s3_points = 64
s4_points = 128
feature_dim = 256
shape_feature_dim = 1024
rnn_step = 4
attention_dim = 32
l1_xyz, l1_points, l1_scales, l1_scales_features, pl_attentions, sl_attention = hierarchical_self_attention(
l0_xyz,
l0_points,
sampled_points, [s1_points, s2_points, s3_points, s4_points],
[[32, feature_dim], [64, feature_dim], [64, feature_dim],
[128, feature_dim]],
attention_dim,
attention_dim,
is_training,
bn_decay,
batch_size=batch_size,
scope='down_layer1')
l4_xyz, latent_representation, l4_indices, rl_attention = mlp_self_attention(
l1_xyz,
l1_points,
npoint=None,
ndim=attention_dim,
nsample=None,
mlp=[256, shape_feature_dim],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='down_layer3')
local_features = pointnet_fp_module(l1_xyz,
l4_xyz,
l1_points,
latent_representation,
[512, feature_dim],
is_training,
bn_decay,
scope='fa_layer1')
'''local branch'''
l1_points = tf_util.rnn_decoder(local_features,
feature_dim,
rnn_step,
scope='decode_layer1',
bn=True,
is_training=is_training,
bn_decay=bn_decay)
scale1_feature = tf.reshape(l1_points[:, :, 0, :], [-1, 1, feature_dim])
scale2_feature = tf.reshape(l1_points[:, :, 1, :], [-1, 1, feature_dim])
scale3_feature = tf.reshape(l1_points[:, :, 2, :], [-1, 1, feature_dim])
scale4_feature = tf.reshape(l1_points[:, :, 3, :], [-1, 1, feature_dim])
# scale1_feature = tf.reshape(l1_points[:, :, 0, :], [-1, 1, 2*feature_dim])
# scale2_feature = tf.reshape(l1_points[:, :, 1, :], [-1, 1, 2*feature_dim])
# scale3_feature = tf.reshape(l1_points[:, :, 2, :], [-1, 1, 2*feature_dim])
# scale4_feature = tf.reshape(l1_points[:, :, 3, :], [-1, 1, 2*feature_dim])
scale1_points = scale_points_generator_fc(scale1_feature,
scale_dim=[s1_points, 3],
layer_sizes=[128, 128],
b_norm=True)
scale2_points = scale_points_generator_fc(scale2_feature,
scale_dim=[s2_points, 3],
layer_sizes=[128, 128],
b_norm=True)
scale3_points = scale_points_generator_fc(scale3_feature,
scale_dim=[s3_points, 3],
layer_sizes=[128, 256],
b_norm=True)
scale4_points = scale_points_generator_fc(scale4_feature,
scale_dim=[s4_points, 3],
layer_sizes=[128, 512],
b_norm=True)
scale_points = tf.concat(
[scale1_points, scale2_points, scale3_points, scale4_points], axis=1)
points_all_features = tf.reshape(scale_points, [batch_size, 1, -1])
generate_points = scale_points_generator_fc(points_all_features,
scale_dim=[num_point, 3],
layer_sizes=[512, 512],
b_norm=True)
scale1_xyz = tf.reshape(l1_scales[0], [-1, s1_points, 3])
scale2_xyz = tf.reshape(l1_scales[1], [-1, s2_points, 3])
scale3_xyz = tf.reshape(l1_scales[2], [-1, s3_points, 3])
scale4_xyz = tf.reshape(l1_scales[3], [-1, s4_points, 3])
scale_points_truth = [scale1_xyz, scale2_xyz, scale3_xyz, scale4_xyz]
scale_points_generate = [
scale1_points, scale2_points, scale3_points, scale4_points
]
end_points['global_feats'] = latent_representation
return generate_points, end_points, scale_points_truth, scale_points_generate, pl_attentions, sl_attention, rl_attention, l1_xyz
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label,
depth=NUM_CATEGORIES,
on_value=1.0,
off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot,
[batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat(
[cls_label_one_hot, l0_xyz, l0_points],
axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
############# PointNet++ 分割部分，先采样，再分组，并分层提取出点云特征######################
batch_size = point_cloud.get_shape()[0].value
print(batch_size)
num_point = point_cloud.get_shape()[1].value
print(num_point)
end_points = {}
l0_xyz = point_cloud[:, :, :3]
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Set abstraction layers, 分层特征提取（加SSG策略，single scale grouping）
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=1024,
radius=0.1,
nsample=32,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
temp1 = l1_points
#print('l1xyz',l1_xyz.shape,'l1point',l1_points.shape,'l1indices',l1_indices.shape)
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=256,
radius=0.2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
temp2 = l2_points
#print('l2xyz', l2_xyz.shape, 'l2point', l2_points.shape, 'l2indices', l2_indices.shape)
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=64,
radius=0.4,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
temp3 = l3_points
#print('l3xyz', l3_xyz.shape, 'l3point', l3_points.shape, 'l3indices', l3_indices.shape)
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=16,
radius=0.8,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
#print('l4xyz', l4_xyz.shape, 'l4point', l4_points.shape, 'l4indices', l4_indices.shape)
# Feature Propagation layers, 对特征进行插值，恢复到之前的点数，最后对每个点进行分类
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l3_points += temp3
#print('l3_point',l3_points.shape)
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
l2_points += temp2
#print('l2_point', l2_points.shape)
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [128, 64],
is_training,
bn_decay,
scope='fa_layer3')
l1_points += temp1
#print('l1_point', l1_points.shape)
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
#print('l0_point', l0_points.shape)
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
#print('conv1d_1',net.shape)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.9,
is_training=is_training,
scope='dp1')
#print('drop',net.shape)
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2') ##(B,L,num_class)
#print('conv1d_2',net.shape)
return net, end_points
def get_displacements(input_points, is_training, noise, FLAGS, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = FLAGS.batch_size
num_points = FLAGS.point_num
point_cloud = input_points
l0_xyz = point_cloud
l0_points = None
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=1024,
radius=0.1 *
FLAGS.radiusScal,
nsample=64,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=384,
radius=0.2 *
FLAGS.radiusScal,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=128,
radius=0.4 *
FLAGS.radiusScal,
nsample=64,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# PointNet
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=None,
radius=None,
nsample=None,
mlp=[512, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
# l4_points = pointnet_fp_module(l4_xyz, l5_xyz, l4_points, l5_points, [512,512], is_training, bn_decay, scope='fa_layer0')
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [512, 512],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [512, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
if noise is not None:
l0_points = tf.concat(axis=2, values=[l0_points, noise])
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.conv1d(net,
64,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.conv1d(net,
3,
1,
padding='VALID',
activation_fn=None,
scope='fc3')
displacements = tf.sigmoid(net) * FLAGS.range_max * 2 - FLAGS.range_max
return displacements
def get_model(point_cloud,
is_training,
num_class,
bn_decay=None,
gripper_feat=None,
env_feat=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=512,
radius=0.01,
nsample=32,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=128,
radius=0.02,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=64,
radius=0.04,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=16,
radius=0.08,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
l5_xyz, l5_points, l5_indices = pointnet_sa_module(l4_xyz,
l4_points,
npoint=4,
radius=0.20,
nsample=32,
mlp=[512, 512, 1024],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer5')
if env_feat is None:
extra_feat = gripper_feat
else:
extra_feat = tf.concat([gripper_feat, env_feat], axis=-1)
print("extra_feat", extra_feat)
extra_feat = tf.expand_dims(extra_feat, axis=1)
extra_feat0 = extra_feat
extra_feat = tflearn.layers.conv.conv_1d(extra_feat,
512,
filter_size=1,
strides=1,
activation=tf.nn.leaky_relu)
extra_feat = tflearn.layers.conv.conv_1d(extra_feat,
256,
filter_size=1,
strides=1,
activation=tf.nn.leaky_relu)
extra_feat = tflearn.layers.conv.conv_1d(extra_feat,
256,
filter_size=1,
strides=1,
activation=tf.nn.leaky_relu)
extra_feat2 = extra_feat
extra_feat = tflearn.layers.conv.conv_1d(extra_feat,
128,
filter_size=1,
strides=1,
activation=tf.nn.leaky_relu)
extra_feat = tflearn.layers.conv.conv_1d(extra_feat,
128,
filter_size=1,
strides=1,
activation=tf.nn.leaky_relu)
extra_feat = tflearn.layers.conv.conv_1d(extra_feat,
128,
filter_size=1,
strides=1,
activation=tf.nn.leaky_relu)
extra_feat = tflearn.layers.conv.conv_1d(extra_feat,
64,
filter_size=1,
strides=1,
activation=tf.nn.leaky_relu)
extra_feat3 = extra_feat
extra_feat4 = extra_feat
extra_feat5 = extra_feat
extra_feat0 = tf.tile(extra_feat0, [1, 4, 1])
l5_points = tf.concat([l5_points, extra_feat0], axis=-1)
l4_points = pointnet_fp_module(l4_xyz,
l5_xyz,
l4_points,
l5_points, [1024, 1024, 512],
is_training,
bn_decay,
scope='fa_layer0',
bn=True)
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [512, 512, 256],
is_training,
bn_decay,
scope='fa_layer1',
bn=True)
extra_feat2 = tf.tile(extra_feat2, [1, 64, 1])
l3_points = tf.concat([l3_points, extra_feat2], axis=-1)
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256, 128],
is_training,
bn_decay,
scope='fa_layer2',
bn=True)
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer3',
bn=True)
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 64],
is_training,
bn_decay,
scope='fa_layer4',
bn=True)
extra_feat5 = tf.tile(extra_feat5, [1, 2048, 1])
# FC layers
net = l0_points #tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=False, is_training=is_training, scope='fc1', bn_decay=bn_decay)
#net = tf_util.conv1d(net, 64, 1, padding='VALID', bn=False, is_training=is_training, scope='fc1_1', bn_decay=bn_decay)
end_points['feats'] = net
#net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
nor = tf_util.conv1d(l0_points,
32,
1,
padding='VALID',
bn=False,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
nor = tf_util.conv1d(nor,
3,
1,
padding='VALID',
bn=False,
is_training=is_training,
scope='fc2_1',
activation_fn=None)
nor = nor / (tf.norm(nor, axis=-1, keep_dims=True) + 1e-10)
#net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
l0_points = tf.concat([l0_points, extra_feat5], axis=-1)
l0_points = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=False,
is_training=is_training,
scope='fc1_3',
bn_decay=bn_decay)
l0_points = tf_util.conv1d(l0_points,
64,
1,
padding='VALID',
bn=False,
is_training=is_training,
scope='fc1_4',
bn_decay=bn_decay)
cla = tf_util.conv1d(l0_points,
num_class,
1,
padding='VALID',
bn=False,
is_training=is_training,
scope='fc3_1',
activation_fn=None)
return cla, nor, end_points, l0_points, extra_feat5
def get_model(point_cloud, is_training, global_config, bn_decay=None):
"""
Contact-GraspNet model consisting of a PointNet++ backbone and multiple output heads
Arguments:
point_cloud {tf.placeholder} -- batch of point clouds
is_training {bool} -- train or eval mode
global_config {dict} -- config
Keyword Arguments:
bn_decay {tf.variable} -- batch norm decay (default: {None})
Returns:
[dict] -- endpoints of the network
"""
model_config = global_config['MODEL']
data_config = global_config['DATA']
radius_list_0 = model_config['pointnet_sa_modules_msg'][0]['radius_list']
radius_list_1 = model_config['pointnet_sa_modules_msg'][1]['radius_list']
radius_list_2 = model_config['pointnet_sa_modules_msg'][2]['radius_list']
nsample_list_0 = model_config['pointnet_sa_modules_msg'][0]['nsample_list']
nsample_list_1 = model_config['pointnet_sa_modules_msg'][1]['nsample_list']
nsample_list_2 = model_config['pointnet_sa_modules_msg'][2]['nsample_list']
mlp_list_0 = model_config['pointnet_sa_modules_msg'][0]['mlp_list']
mlp_list_1 = model_config['pointnet_sa_modules_msg'][1]['mlp_list']
mlp_list_2 = model_config['pointnet_sa_modules_msg'][2]['mlp_list']
npoint_0 = model_config['pointnet_sa_modules_msg'][0]['npoint']
npoint_1 = model_config['pointnet_sa_modules_msg'][1]['npoint']
npoint_2 = model_config['pointnet_sa_modules_msg'][2]['npoint']
fp_mlp_0 = model_config['pointnet_fp_modules'][0]['mlp']
fp_mlp_1 = model_config['pointnet_fp_modules'][1]['mlp']
fp_mlp_2 = model_config['pointnet_fp_modules'][2]['mlp']
input_normals = data_config['input_normals']
offset_bins = data_config['labels']['offset_bins']
joint_heads = model_config['joint_heads']
# expensive, rather use random only
if 'raw_num_points' in data_config and data_config[
'raw_num_points'] != data_config['ndataset_points']:
point_cloud = gather_point(
point_cloud,
farthest_point_sample(data_config['ndataset_points'], point_cloud))
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3],
[-1, -1, 3]) if input_normals else None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz,
l0_points,
npoint_0,
radius_list_0,
nsample_list_0,
mlp_list_0,
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz,
l1_points,
npoint_1,
radius_list_1,
nsample_list_1,
mlp_list_1,
is_training,
bn_decay,
scope='layer2')
if 'asymmetric_model' in model_config and model_config['asymmetric_model']:
l3_xyz, l3_points = pointnet_sa_module_msg(l2_xyz,
l2_points,
npoint_2,
radius_list_2,
nsample_list_2,
mlp_list_2,
is_training,
bn_decay,
scope='layer3')
l4_xyz, l4_points, _ = pointnet_sa_module(
l3_xyz,
l3_points,
npoint=None,
radius=None,
nsample=None,
mlp=model_config['pointnet_sa_module']['mlp'],
mlp2=None,
group_all=model_config['pointnet_sa_module']['group_all'],
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points,
fp_mlp_0,
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points,
fp_mlp_1,
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points,
fp_mlp_2,
is_training,
bn_decay,
scope='fa_layer3')
l0_points = l1_points
pred_points = l1_xyz
else:
l3_xyz, l3_points, _ = pointnet_sa_module(
l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=model_config['pointnet_sa_module']['mlp'],
mlp2=None,
group_all=model_config['pointnet_sa_module']['group_all'],
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points,
fp_mlp_0,
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points,
fp_mlp_1,
is_training,
bn_decay,
scope='fa_layer2')
l0_points = tf.concat([l0_xyz, l0_points],
axis=-1) if input_normals else l0_xyz
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points,
fp_mlp_2,
is_training,
bn_decay,
scope='fa_layer3')
pred_points = l0_xyz
if joint_heads:
head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
head = tf_util.dropout(head,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
head = tf_util.conv1d(head,
4,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
grasp_dir_head = tf.slice(head, [0, 0, 0], [-1, -1, 3])
grasp_dir_head = tf.math.l2_normalize(grasp_dir_head, axis=2)
binary_seg_head = tf.slice(head, [0, 0, 3], [-1, -1, 1])
else:
# Head for grasp direction
grasp_dir_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
grasp_dir_head = tf_util.dropout(grasp_dir_head,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
grasp_dir_head = tf_util.conv1d(grasp_dir_head,
3,
1,
padding='VALID',
activation_fn=None,
scope='fc3')
grasp_dir_head_normed = tf.math.l2_normalize(grasp_dir_head, axis=2)
# Head for grasp approach
approach_dir_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_app',
bn_decay=bn_decay)
approach_dir_head = tf_util.dropout(approach_dir_head,
keep_prob=0.7,
is_training=is_training,
scope='dp1_app')
approach_dir_head = tf_util.conv1d(approach_dir_head,
3,
1,
padding='VALID',
activation_fn=None,
scope='fc3_app')
approach_dir_head_orthog = tf.math.l2_normalize(
approach_dir_head - tf.reduce_sum(
tf.multiply(grasp_dir_head_normed, approach_dir_head),
axis=2,
keepdims=True) * grasp_dir_head_normed,
axis=2)
# Head for grasp width
if model_config['dir_vec_length_offset']:
grasp_offset_head = tf.norm(grasp_dir_head, axis=2, keepdims=True)
elif model_config['bin_offsets']:
grasp_offset_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_off',
bn_decay=bn_decay)
grasp_offset_head = tf_util.conv1d(grasp_offset_head,
len(offset_bins) - 1,
1,
padding='VALID',
activation_fn=None,
scope='fc2_off')
else:
grasp_offset_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_off',
bn_decay=bn_decay)
grasp_offset_head = tf_util.dropout(grasp_offset_head,
keep_prob=0.7,
is_training=is_training,
scope='dp1_off')
grasp_offset_head = tf_util.conv1d(grasp_offset_head,
1,
1,
padding='VALID',
activation_fn=None,
scope='fc2_off')
# Head for contact points
binary_seg_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_seg',
bn_decay=bn_decay)
binary_seg_head = tf_util.dropout(binary_seg_head,
keep_prob=0.5,
is_training=is_training,
scope='dp1_seg')
binary_seg_head = tf_util.conv1d(binary_seg_head,
1,
1,
padding='VALID',
activation_fn=None,
scope='fc2_seg')
end_points['grasp_dir_head'] = grasp_dir_head_normed
end_points['binary_seg_head'] = binary_seg_head
end_points['binary_seg_pred'] = tf.math.sigmoid(binary_seg_head)
end_points['grasp_offset_head'] = grasp_offset_head
end_points['grasp_offset_pred'] = tf.math.sigmoid(
grasp_offset_head
) if model_config['bin_offsets'] else grasp_offset_head
end_points['approach_dir_head'] = approach_dir_head_orthog
end_points['pred_points'] = pred_points
return end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
############# PointNet++ 分割部分，先采样，再分组，并分层提取出点云特征######################
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud[:, :, :3]
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Set abstraction layers, 分层特征提取（加SSG策略，single scale grouping）
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=1024,
radius=0.1,
nsample=32,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l01_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [512],
is_training,
bn_decay,
scope='fa_layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=256,
radius=0.2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l02_points = pointnet_fp_module(l0_xyz,
l2_xyz,
l0_points,
l2_points, [512],
is_training,
bn_decay,
scope='fa_layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=64,
radius=0.4,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l03_points = pointnet_fp_module(l0_xyz,
l3_xyz,
l0_points,
l3_points, [512],
is_training,
bn_decay,
scope='fa_layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=16,
radius=0.8,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
l04_points = pointnet_fp_module(l0_xyz,
l4_xyz,
l0_points,
l4_points, [512],
is_training,
bn_decay,
scope='fa_layer4')
# Feature Propagation layers, 对特征进行插值，恢复到之前的点数，最后对每个点进行分类
# l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
# l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
# l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
# l0_points = pointnet_fp_module(l0_xyz, l1_xyz, l0_points, l1_points, [128, 128, 128], is_training, bn_decay,scope='fa_layer4')
# l0_points = pointnet_fp_module(l0_xyz, l4_xyz, l0_points, l4_points, [128, 128, 128], is_training, bn_decay,scope='fa_layer4')
# l0_points = pointnet_fp_module(l0_xyz, l4_xyz, l0_points, l4_points, [128, 128, 128], is_training, bn_decay,scope='fa_layer4')
# l0_points = pointnet_fp_module(l0_xyz, l4_xyz, l0_points, l4_points, [128, 128, 128], is_training, bn_decay,scope='fa_layer4')
# FC layers
l0_points = tf.concat([l01_points, l02_points, l03_points, l04_points],
axis=-1) #2048
l0_points = tf_util.dropout(l0_points,
keep_prob=0.8,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(l0_points,
512,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.8,
is_training=is_training,
scope='dp2')
net = tf_util.conv1d(net,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.8,
is_training=is_training,
scope='dp3')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc3') ##(B,L,num_class)
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3]) # point coordinates
l0_points = tf.slice(point_cloud, [0, 0, 3],
[-1, -1, -1]) #point attributes
end_points['l0_xyz'] = l0_xyz
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=50000,
radius=1,
nsample=32,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=10000,
radius=2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=2000,
radius=4,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=500,
radius=9,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
l5_xyz, l5_points, l5_indices = pointnet_sa_module(l4_xyz,
l4_points,
npoint=100,
radius=25,
nsample=32,
mlp=[512, 512, 1024],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer5')
# debug line:
# l4_points = tf.Print(l1_points, [l0_xyz, l0_points, l1_xyz, l1_points], 'ln-points', -1, 12)
end_points['l1_xyz'] = l1_xyz
end_points['l2_xyz'] = l2_xyz
end_points['l3_xyz'] = l3_xyz
end_points['l4_xyz'] = l4_xyz
end_points['l5_xyz'] = l5_xyz
# Feature Propagation layers
l4_points = pointnet_fp_module(l4_xyz,
l5_xyz,
l4_points,
l5_points, [512, 512],
is_training,
bn_decay,
scope='fa_layer0')
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
num_class,
1,
padding='VALID',
activation_fn=None,
scope='fc2',
name='net')
return net, end_points