def __init__(self, points, features, is_training, setting):
bn_decay = setting.get_bn_decay(tf.train.get_global_step())
l0_xyz = points
l0_points = None
num_class = setting.num_class
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [64, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training,
bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l3_input_shape, l2_xyz, l2_points, npoint=None, radius=None,
nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True,
is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [l3_input_shape[0], -1])
net = tf_util.fully_connected(FC1_inputs_shape, net, 512, bn=True, is_training=is_training, scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(FC2_inputs_shape, net, 256, bn=True, is_training=is_training, scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(FC3_inputs_shape, net, num_class, activation_fn=None, scope='fc3')
self.logits = tf.expand_dims(net,axis = 1)
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1])
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, num_class, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx5, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,2])
end_points['l0_xyz'] = l0_xyz
# Layer 1
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 1024, [0.1, 0.4], [16, 128],[[32,64],[64, 128],[64,128]], is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 256, [0.1, 0.5], [32,128],[[64,128], [128,256], [128,256]], is_training, bn_decay=bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, 128, radius=0.6, nsample=64, mlp=[256,128,256],mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz, l3_points, 16, radius=0.8, nsample=32, mlp=[256,256,512],mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz, l4_xyz, l3_points, l4_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz,l0_points],axis=-1), l1_points, [128,128,128], is_training, bn_decay, scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, num_class, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, scope='', num_point=None, bn_decay=None, ifglobal=False, bn=True, end_points={}):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [16,32,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1', use_nchw=True, bn = bn)
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2,0.4,0.8], [32,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2', bn = bn)
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3', bn = bn)
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=bn, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['embedding'] = net
# net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1')
# net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
# net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2')
# net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
end_points['l2_xyz'] = l2_xyz
end_points['l3_xyz'] = l3_xyz
end_points['l1_xyz'] = l1_xyz
end_points['l0_xyz'] = l0_xyz
end_points['l2_points'] = l2_points
end_points['l3_points'] = l3_points
end_points['l1_points'] = l1_points
end_points['l0_points'] = l0_points
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
# num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512,
[0.1, 0.2, 0.4],
[16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope="layer1",
use_nchw=True,
)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128,
[0.2, 0.4, 0.8],
[32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope="layer2",
)
l3_xyz, l3_points, _ = pointnet_sa_module(
l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope="layer3",
)
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(
net, 512, bn=True, is_training=is_training, scope="fc1", bn_decay=bn_decay
)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope="dp1")
net = tf_util.fully_connected(
net, 256, bn=True, is_training=is_training, scope="fc2", bn_decay=bn_decay
)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope="dp2")
net = tf_util.fully_connected(net, 40, activation_fn=None, scope="fc3")
return net, end_points
def get_instance_seg_v2_net(point_cloud, one_hot_vec,
is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points,
128, [0.2,0.4,0.8], [32,64,128],
[[32,32,64], [64,64,128], [64 ,96,128]],
is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points,
32, [0.4,0.8,1.6], [64,64,128],
[[64,64,128], [128,128,256], [128,128,256]],
is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points,
npoint=None, radius=None, nsample=None, mlp=[128,256,1024],
mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2)
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points,
[128,128], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points,
[128,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz,
tf.concat([l0_xyz,l0_points],axis=-1), l1_points,
[128,128], is_training, bn_decay, scope='fa_layer3')
end_points['feats'] = l0_points
# FC layers
#print("l0_points",l0_points.shape)
#print("l0_points",l0_points)
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True,
is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay)
#print("l0_points",l0_points.shape)
net = tf_util.dropout(net, keep_prob=0.7,
is_training=is_training, scope='dp1')
logits = tf_util.conv1d(net, 2, 1,
padding='VALID', activation_fn=None, scope='conv1d-fc2')
return logits, end_points
def get_instance_seg_v2_net(point_cloud, one_hot_vec,
is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points,
128, [0.2,0.4,0.8], [32,64,128],
[[32,32,64], [64,64,128], [64,96,128]],
is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points,
32, [0.4,0.8,1.6], [64,64,128],
[[64,64,128], [128,128,256], [128,128,256]],
is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points,
npoint=None, radius=None, nsample=None, mlp=[128,256,1024],
mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
# Feature Propagation layers
l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2)
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points,
[128,128], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points,
[128,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz,
tf.concat([l0_xyz,l0_points],axis=-1), l1_points,
[128,128], is_training, bn_decay, scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True,
is_training=is_training, scope='conv1d-fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.7,
is_training=is_training, scope='dp1')
logits = tf_util.conv1d(net, 2, 1,
padding='VALID', activation_fn=None, scope='conv1d-fc2')
return logits, end_points
def get_features(point_cloud, out_dim):
is_training = tf.constant(True)
bn_decay = None
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = tf.shape(point_cloud)[0]
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
4, [0.1, 0.2, 0.4], [4, 4, 4],
[[16, 16, 32], [16, 16, 32], [32, 64, 64]],
is_training,
bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
2, [0.2, 0.4, 0.8], [2, 2, 2],
[[8, 8, 16], [16, 16, 32], [16, 32, 32]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[32, 32, 64],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [-1, l3_points.shape[1] * l3_points.shape[2]])
# net = fully_connected(net, 64, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
# net = dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1')
# net = fully_connected(net, 64, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
# net = dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2')
net = fully_connected(net, out_dim, activation_fn=None, scope='fc3')
return net
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3])
l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [32,64,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.4,0.8], [64,128], [[128,128,256],[128,196,256]], is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [256,256], is_training, bn_decay, scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label, depth=NUM_CATEGORIES, on_value=1.0, off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot, [batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1,num_point,1])
# Calculate the relative coordinates generated in the first set abstraction layer
# These relative coordinates will be concatenated in the last feature propagation layer
dist, idx = three_nn(l0_xyz, l1_xyz)
dist = tf.maximum(dist, 1e-10)
norm = tf.reduce_sum((1.0/dist),axis=2,keep_dims=True)
norm = tf.tile(norm,[1,1,3])
weight = (1.0/dist) / norm
l0_virtual_centers = three_interpolate(l1_xyz, idx, weight)
l0_xyz_rel = l0_xyz - l0_virtual_centers
#l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_xyz_rel, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([cls_label_one_hot, l0_points],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 50, 1, padding='VALID', activation_fn=None, scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, batchnorm=False, bn_decay=None, dropout_rate=0.1):
"""
PointNet++ for TAWSS regression, input is BxNxF, output BxN
"""
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
num_features = point_cloud.get_shape()[2].value
l0_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) # point coordinates
if num_features == 3: l0_points = None
else: l0_points = tf.slice(point_cloud, [0,0,3], [-1,-1,1]) # scale information
#mid_xyz = {'l0_xyz': l0_xyz}
#mid_points = {'l0_points': l0_points}
# Set Abstraction layers with multi-scale grouping
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 256, [0.1,0.2,0.4], [16,32,64], [[64,128], [128,128], [128,128]], is_training, bn_decay, dropout_rate, scope='layer1', bn=batchnorm)
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 16, [0.2,0.4,0.8], [16,32,64], [[128],[256],[512]], is_training, bn_decay, dropout_rate, scope='layer2', bn=batchnorm)
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3', bn=batchnorm)
#mid_xyz['l2_xyz'] = l2_xyz
#mid_points['l2_points'] = l2_points
#mid_xyz['l1_xyz'] = l1_xyz
#mid_points['l1_points'] = l1_points
#mid_xyz['l3_xyz'] = l3_xyz
#mid_points['l3_points'] = l3_points
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz, l3_xyz, l2_points, l3_points, [512], is_training, bn_decay, dropout_rate, scope='fp_layer1', bn=batchnorm)
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256], is_training, bn_decay, dropout_rate, scope='fp_layer2', bn=batchnorm)
l0_points = pointnet_fp_module(l0_xyz, l1_xyz, tf.concat([l0_xyz, l0_points], axis=-1), l1_points, [128], is_training, bn_decay, dropout_rate, scope='fp_layer3', bn=batchnorm)
# Fully Connected layers
net = tf_util.conv1d(l0_points, 128, 1, scope='fc1', padding='VALID', is_training=is_training, bn=batchnorm, bn_decay=bn_decay)
#mid_points['feats'] = net
net = tf_util.dropout(net, rate=dropout_rate, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 1, 1, scope='fc2', padding='VALID', activation_fn=None, bn=False)
return net#, mid_xyz, mid_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz, l0_points, 512, [0.1,0.2,0.4], [16,32,128], [[32,32,64], [64,64,128], [64,96,128]], is_training, bn_decay, scope='layer1', use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz, l1_points, 128, [0.2,0.4,0.8], [32,64,128], [[64,64,128], [128,128,256], [128,128,256]], is_training, bn_decay, scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256,512,1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_proposal_cls_net(point_cloud, img_seg_map, is_training, bn_decay,
end_points):
batch_size = point_cloud.shape[0]
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, NUM_CHANNEL - 3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='cls_layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
32, [0.4, 0.8, 1.6], [64, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='cls_layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[128, 256, 512],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='cls_layer3')
# image feature pooling
_img_pixel_size = np.asarray([360, 1200])
box2d_corners, box2d_corners_norm = projection.tf_project_to_image_space(
proposal_boxes, calib, _img_pixel_size)
# crop and resize
'''
# y1, x1, y2, x2
box2d_corners_norm_reorder = tf.stack([
tf.gather(box2d_corners_norm, 1, axis=-1),
tf.gather(box2d_corners_norm, 0, axis=-1),
tf.gather(box2d_corners_norm, 3, axis=-1),
tf.gather(box2d_corners_norm, 2, axis=-1),
], axis=-1)
img_rois = tf.image.crop_and_resize(
img_seg_map,
box2d_corners_norm_reorder,
tf.range(0, batch_size),
[16,16])
'''
# roi pooling in faster-rcnn
img_seg_map = tf_util.conv2d(img_seg_map,
1,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='cls_feature_bottleneck',
bn_decay=bn_decay)
# feature map index, upper left, bottom right coordinates
roi_crops = tf.concat(tf.expand_dims(tf.range(0, batch_size), axis=-1),
box2d_corners,
axis=-1)
img_rois = roi_pooling(img_seg_map,
roi_crops,
pool_height=16,
pool_width=16)
img_feats = tf.reshape(img_rois, [batch_size, -1])
# classification
point_feats = tf.reshape(l3_points, [batch_size, -1])
# use image only
#cls_net = img_feats
# use point and image feature
cls_net = tf.concat([point_feats, img_feats], axis=1)
# use point only
#cls_net = point_feats
cls_net = tf_util.fully_connected(cls_net,
512,
bn=True,
is_training=is_training,
scope='cls_fc1',
bn_decay=bn_decay)
cls_net = tf_util.dropout(cls_net,
keep_prob=0.5,
is_training=is_training,
scope='cls_dp1')
cls_net = tf_util.fully_connected(cls_net,
256,
bn=True,
is_training=is_training,
scope='cls_fc2',
bn_decay=bn_decay)
cls_net = tf_util.dropout(cls_net,
keep_prob=0.5,
is_training=is_training,
scope='cls_dp2')
cls_net = tf_util.fully_connected(cls_net,
NUM_OBJ_CLASSES,
activation_fn=None,
scope='cls_logits')
end_points['cls_logits'] = cls_net
return end_points
def get_model(point_cloud, is_training, global_config, bn_decay=None):
"""
Contact-GraspNet model consisting of a PointNet++ backbone and multiple output heads
Arguments:
point_cloud {tf.placeholder} -- batch of point clouds
is_training {bool} -- train or eval mode
global_config {dict} -- config
Keyword Arguments:
bn_decay {tf.variable} -- batch norm decay (default: {None})
Returns:
[dict] -- endpoints of the network
"""
model_config = global_config['MODEL']
data_config = global_config['DATA']
radius_list_0 = model_config['pointnet_sa_modules_msg'][0]['radius_list']
radius_list_1 = model_config['pointnet_sa_modules_msg'][1]['radius_list']
radius_list_2 = model_config['pointnet_sa_modules_msg'][2]['radius_list']
nsample_list_0 = model_config['pointnet_sa_modules_msg'][0]['nsample_list']
nsample_list_1 = model_config['pointnet_sa_modules_msg'][1]['nsample_list']
nsample_list_2 = model_config['pointnet_sa_modules_msg'][2]['nsample_list']
mlp_list_0 = model_config['pointnet_sa_modules_msg'][0]['mlp_list']
mlp_list_1 = model_config['pointnet_sa_modules_msg'][1]['mlp_list']
mlp_list_2 = model_config['pointnet_sa_modules_msg'][2]['mlp_list']
npoint_0 = model_config['pointnet_sa_modules_msg'][0]['npoint']
npoint_1 = model_config['pointnet_sa_modules_msg'][1]['npoint']
npoint_2 = model_config['pointnet_sa_modules_msg'][2]['npoint']
fp_mlp_0 = model_config['pointnet_fp_modules'][0]['mlp']
fp_mlp_1 = model_config['pointnet_fp_modules'][1]['mlp']
fp_mlp_2 = model_config['pointnet_fp_modules'][2]['mlp']
input_normals = data_config['input_normals']
offset_bins = data_config['labels']['offset_bins']
joint_heads = model_config['joint_heads']
# expensive, rather use random only
if 'raw_num_points' in data_config and data_config[
'raw_num_points'] != data_config['ndataset_points']:
point_cloud = gather_point(
point_cloud,
farthest_point_sample(data_config['ndataset_points'], point_cloud))
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3],
[-1, -1, 3]) if input_normals else None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(l0_xyz,
l0_points,
npoint_0,
radius_list_0,
nsample_list_0,
mlp_list_0,
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(l1_xyz,
l1_points,
npoint_1,
radius_list_1,
nsample_list_1,
mlp_list_1,
is_training,
bn_decay,
scope='layer2')
if 'asymmetric_model' in model_config and model_config['asymmetric_model']:
l3_xyz, l3_points = pointnet_sa_module_msg(l2_xyz,
l2_points,
npoint_2,
radius_list_2,
nsample_list_2,
mlp_list_2,
is_training,
bn_decay,
scope='layer3')
l4_xyz, l4_points, _ = pointnet_sa_module(
l3_xyz,
l3_points,
npoint=None,
radius=None,
nsample=None,
mlp=model_config['pointnet_sa_module']['mlp'],
mlp2=None,
group_all=model_config['pointnet_sa_module']['group_all'],
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points,
fp_mlp_0,
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points,
fp_mlp_1,
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points,
fp_mlp_2,
is_training,
bn_decay,
scope='fa_layer3')
l0_points = l1_points
pred_points = l1_xyz
else:
l3_xyz, l3_points, _ = pointnet_sa_module(
l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=model_config['pointnet_sa_module']['mlp'],
mlp2=None,
group_all=model_config['pointnet_sa_module']['group_all'],
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points,
fp_mlp_0,
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points,
fp_mlp_1,
is_training,
bn_decay,
scope='fa_layer2')
l0_points = tf.concat([l0_xyz, l0_points],
axis=-1) if input_normals else l0_xyz
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points,
fp_mlp_2,
is_training,
bn_decay,
scope='fa_layer3')
pred_points = l0_xyz
if joint_heads:
head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
head = tf_util.dropout(head,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
head = tf_util.conv1d(head,
4,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
grasp_dir_head = tf.slice(head, [0, 0, 0], [-1, -1, 3])
grasp_dir_head = tf.math.l2_normalize(grasp_dir_head, axis=2)
binary_seg_head = tf.slice(head, [0, 0, 3], [-1, -1, 1])
else:
# Head for grasp direction
grasp_dir_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
grasp_dir_head = tf_util.dropout(grasp_dir_head,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
grasp_dir_head = tf_util.conv1d(grasp_dir_head,
3,
1,
padding='VALID',
activation_fn=None,
scope='fc3')
grasp_dir_head_normed = tf.math.l2_normalize(grasp_dir_head, axis=2)
# Head for grasp approach
approach_dir_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_app',
bn_decay=bn_decay)
approach_dir_head = tf_util.dropout(approach_dir_head,
keep_prob=0.7,
is_training=is_training,
scope='dp1_app')
approach_dir_head = tf_util.conv1d(approach_dir_head,
3,
1,
padding='VALID',
activation_fn=None,
scope='fc3_app')
approach_dir_head_orthog = tf.math.l2_normalize(
approach_dir_head - tf.reduce_sum(
tf.multiply(grasp_dir_head_normed, approach_dir_head),
axis=2,
keepdims=True) * grasp_dir_head_normed,
axis=2)
# Head for grasp width
if model_config['dir_vec_length_offset']:
grasp_offset_head = tf.norm(grasp_dir_head, axis=2, keepdims=True)
elif model_config['bin_offsets']:
grasp_offset_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_off',
bn_decay=bn_decay)
grasp_offset_head = tf_util.conv1d(grasp_offset_head,
len(offset_bins) - 1,
1,
padding='VALID',
activation_fn=None,
scope='fc2_off')
else:
grasp_offset_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_off',
bn_decay=bn_decay)
grasp_offset_head = tf_util.dropout(grasp_offset_head,
keep_prob=0.7,
is_training=is_training,
scope='dp1_off')
grasp_offset_head = tf_util.conv1d(grasp_offset_head,
1,
1,
padding='VALID',
activation_fn=None,
scope='fc2_off')
# Head for contact points
binary_seg_head = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1_seg',
bn_decay=bn_decay)
binary_seg_head = tf_util.dropout(binary_seg_head,
keep_prob=0.5,
is_training=is_training,
scope='dp1_seg')
binary_seg_head = tf_util.conv1d(binary_seg_head,
1,
1,
padding='VALID',
activation_fn=None,
scope='fc2_seg')
end_points['grasp_dir_head'] = grasp_dir_head_normed
end_points['binary_seg_head'] = binary_seg_head
end_points['binary_seg_pred'] = tf.math.sigmoid(binary_seg_head)
end_points['grasp_offset_head'] = grasp_offset_head
end_points['grasp_offset_pred'] = tf.math.sigmoid(
grasp_offset_head
) if model_config['bin_offsets'] else grasp_offset_head
end_points['approach_dir_head'] = approach_dir_head_orthog
end_points['pred_points'] = pred_points
return end_points
def vlad_forward(xyz, reshaped_input, feature_size=1024, max_samples=4096, cluster_size=64,
output_dim=256, gating=True, add_batch_norm=True,
is_training=True, bn_decay=None):
"""Forward pass of a NetVLAD block.
Args:
reshaped_input: If your input is in that form:
'batch_size' x 'max_samples' x 'feature_size'
It should be reshaped in the following form:
'batch_size*max_samples' x 'feature_size'
by performing:
reshaped_input = tf.reshape(input, [-1, features_size])
Returns:
vlad: the pooled vector of size: 'batch_size' x 'output_dim'
"""
input = tf.reshape(reshaped_input, [-1,
max_samples, feature_size])
#msg grouping
l1_xyz, l1_points = pointnet_sa_module_msg(xyz, input, 256, [0.1, 0.2, 0.4], [16, 32, 64],
[[16, 16, 32], [32, 32, 64], [32, 64, 64]], is_training,
bn_decay,
scope='layer1', use_nchw=True)
l2_xyz, l2_points, _ = pointnet_sa_module(l1_xyz, l1_points, npoint=None, radius=None, nsample=None,
mlp=[256, 512], mlp2=None, group_all=True, is_training=is_training,
bn_decay=bn_decay, scope='layer3')
print('l2_points:', l2_points)
l1_points = pointnet_fp_module(l1_xyz, l2_xyz, l1_points, l2_points, [256,128], is_training, bn_decay, scope='fa_layer2')
l0_points = pointnet_fp_module(xyz, l1_xyz, tf.concat([xyz,input],axis=-1), l1_points, [128,128], is_training, bn_decay, scope='fa_layer3')
print('l0_points shape', l0_points)
net = tf_util.conv1d(l0_points, 1, 1, padding='VALID', bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
net = tf_util.conv1d(net, 1, 1, padding='VALID', activation_fn=None, scope='fc2')
m = tf.reshape(net, [-1, 1])
print('m:', m)
#constrain weights to [0, 1]
m = tf.nn.sigmoid(m)
weights = m
m = tf.tile(m, [1, cluster_size])
print('m:', m)
cluster_weights = tf.get_variable("cluster_weights",
[feature_size, cluster_size],
initializer=tf.random_normal_initializer(
stddev=1 / math.sqrt(feature_size)))
activation = tf.matmul(reshaped_input, cluster_weights)
# activation = tf.contrib.layers.batch_norm(activation,
# center=True, scale=True,
# is_training=self.is_training,
# scope='cluster_bn')
# activation = slim.batch_norm(
# activation,
# center=True,
# scale=True,
# is_training=self.is_training,
# scope="cluster_bn")
if add_batch_norm:
activation = slim.batch_norm(
activation,
center=True,
scale=True,
is_training=is_training,
scope="cluster_bn", fused=False)
else:
cluster_biases = tf.get_variable("cluster_biases",
[cluster_size],
initializer=tf.random_normal_initializer(
stddev=1 / math.sqrt(feature_size)))
activation = activation + cluster_biases
activation = tf.nn.softmax(activation)
activation_crn = tf.multiply(activation, m)
activation = tf.reshape(activation_crn,
[-1, max_samples, cluster_size])
a_sum = tf.reduce_sum(activation, -2, keepdims=True)
cluster_weights2 = tf.get_variable("cluster_weights2",
[1, feature_size, cluster_size],
initializer=tf.random_normal_initializer(
stddev=1 / math.sqrt(feature_size)))
a = tf.multiply(a_sum, cluster_weights2)
activation = tf.transpose(activation, perm=[0, 2, 1])
reshaped_input = tf.reshape(reshaped_input, [-1,
max_samples, feature_size])
vlad = tf.matmul(activation, reshaped_input)
vlad = tf.transpose(vlad, perm=[0, 2, 1])
vlad = tf.subtract(vlad, a)
vlad = tf.nn.l2_normalize(vlad, 1)
vlad = tf.reshape(vlad, [-1, cluster_size * feature_size])
vlad = tf.nn.l2_normalize(vlad, 1)
hidden1_weights = tf.get_variable("hidden1_weights",
[cluster_size * feature_size, output_dim],
initializer=tf.random_normal_initializer(
stddev=1 / math.sqrt(cluster_size)))
##Tried using dropout
# vlad=tf.layers.dropout(vlad,rate=0.5,training=self.is_training)
vlad = tf.matmul(vlad, hidden1_weights)
##Added a batch norm
vlad = tf.contrib.layers.batch_norm(vlad,
center=True, scale=True,
is_training=is_training,
scope='bn')
if gating:
vlad = context_gating(vlad, add_batch_norm, is_training)
return vlad, weights
def get_segmentation_net(self, point_cloud, is_training, bn_decay,
end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, NUM_CHANNEL - 3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
4096, [0.1, 0.5], [16, 32], [[16, 16, 32], [32, 32, 64]],
is_training,
bn_decay,
scope='layer1',
bn=True)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
1024, [0.5, 1.0], [16, 32], [[64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer2',
bn=True)
l3_xyz, l3_points = pointnet_sa_module_msg(
l2_xyz,
l2_points,
256, [1.0, 2.0], [16, 32], [[128, 196, 256], [128, 196, 256]],
is_training,
bn_decay,
scope='layer3',
bn=True)
l4_xyz, l4_points = pointnet_sa_module_msg(
l3_xyz,
l3_points,
64, [2.0, 4.0], [16, 32], [[256, 256, 512], [256, 384, 512]],
is_training,
bn_decay,
scope='layer4',
bn=True)
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [512, 512],
is_training,
bn_decay,
scope='fa_layer2',
bn=True)
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [512, 512],
is_training,
bn_decay,
scope='fa_layer3',
bn=True)
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer4',
bn=True)
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat([l0_xyz, l0_points], axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer5',
bn=True)
end_points['point_feats'] = tf.concat([l0_xyz, l0_points],
axis=-1) # (B, N, 3+C1)
end_points['point_feats_fuse'] = tf.concat(
[end_points['point_feats'], end_points['point_img_feats']],
axis=-1) # (B, N, 3+C1+C2)
semantic_features = tf.concat(
[l0_points, end_points['point_img_feats']],
axis=-1) # (B, N, C1+C2)
#end_points['point_feats_fuse'] = end_points['point_feats']
#semantic_features = l0_points
# FC layers
net = tf_util.dropout(semantic_features,
keep_prob=0.5,
is_training=is_training,
scope='dp0')
net = tf_util.conv1d(net,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='conv1d-fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
logits = tf_util.conv1d(net,
NUM_SEG_CLASSES,
1,
padding='VALID',
activation_fn=None,
scope='conv1d-fc2')
end_points['foreground_logits'] = logits
return end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
# Set abstraction layers
# Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4).
# So we only use NCHW for layer 1 until this issue can be resolved.
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
4096, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
1024, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
12800,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
6400,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp2')
attachment_net = tf_util.fully_connected(net,
128 * 40,
activation_fn=None,
scope='fc3')
type_net = tf_util.fully_connected(net,
128 * 4,
activation_fn=None,
scope='fc4')
orientation_net = tf_util.fully_connected(net,
128 * 4,
activation_fn=None,
scope='fc5')
surface_net = tf_util.fully_connected(net,
128 * 4,
activation_fn=None,
scope='fc6')
startstage_net = tf_util.fully_connected(net,
128 * 100,
activation_fn=None,
scope='fc7')
attachment_net = tf.reshape(attachment_net, [batch_size, 128, -1])
type_net = tf.reshape(type_net, [batch_size, 128, -1])
orientation_net = tf.reshape(orientation_net, [batch_size, 128, -1])
surface_net = tf.reshape(surface_net, [batch_size, 128, -1])
startstage_net = tf.reshape(startstage_net, [batch_size, 128, -1])
return attachment_net, type_net, orientation_net, surface_net, startstage_net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
# Apply input-transform network
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3,
normals=True)
l0_xyz_transformed = tf.matmul(l0_xyz, transform)
l0_points_transformed = tf.matmul(l0_points, transform)
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz_transformed,
l0_points_transformed,
512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1')
# Apply feature-transform network after first SA layer
with tf.variable_scope('transform_net2') as sc:
K = l1_points.get_shape(
)[2].value # features were concatenated from multi-scales
l1_points_expanded = tf.expand_dims(l1_points, 2)
transform = feature_transform_net(l1_points_expanded,
is_training,
bn_decay,
K=K)
l1_points_transformed = tf.matmul(l1_points, transform)
end_points['transform'] = transform
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points_transformed,
128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label,
depth=NUM_CATEGORIES,
on_value=1.0,
off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot,
[batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(
l0_xyz_transformed,
l1_xyz,
tf.concat(
[cls_label_one_hot, l0_xyz_transformed, l0_points_transformed],
axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, label, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud
l0_points = None
ppc = []
l0_xyz, l0_points = pointnet_sa_module_msg(l0_xyz[:,:,0:3], l0_points, 512,
radius_list=[0.1,0.2,0.4,0.8],
nsample_list=[8,16,32,64],
mlp_list=[[32,32,64], [32,32,64], [32,32,64], [32,32,64]],
is_training=is_training,
bn_decay=bn_decay,
knn=True,
scope='layer1',
use_nchw=False,
rearrange=True)
l0_xyz, l0_points = pointnet_sa_module_msg(l0_xyz[:,:,0:3], l0_points, 256,
radius_list=[0.1,0.2,0.4,0.8],
nsample_list=[8,16,32,64],
mlp_list=[[64,64,128], [64,64,128], [64,64,128], [64,64,128]],
is_training=is_training,
bn_decay=bn_decay,
knn=True,
scope='layer0',
use_nchw=False,
rearrange=True)
l0_points = tf.reshape(l0_points, [batch_size, 512, 256])
print(l0_points.get_shape)
with tf.variable_scope('netVLAD_feats'):
l2_points = netVLAD(l0_points, 64, is_training=is_training, bn=True, bn_decay=bn_decay)
with tf.variable_scope('netVLAD_xyz'):
l2_xyz = netVLAD(l0_xyz, 64, is_training=is_training, bn=True, bn_decay=bn_decay)
l2_points = tf.concat([l2_points, l2_xyz], axis=-1)
l2_points = tf.contrib.layers.fully_connected(inputs=l2_points, num_outputs=256, scope='NetVLAD_fc',biases_initializer=None)
l2_points = slim.batch_norm(l2_points,
center=True,
scale=True,
is_training=is_training,
scope="cluster_bn", fused=False, decay=bn_decay)
ppc = tf.reshape(l2_points, (batch_size, 1, 1024, 16, 1))
ppc = squashing(ppc)
ppc = tf.squeeze(ppc)
with tf.variable_scope('DigitCaps_Layers_1'):
digitCaps = CapsLayer(input_number=1024, output_number=NUM_CLASSES, vec_length=16, out_length=32, layer_type='FC')
caps = digitCaps(ppc, is_training=is_training, bn=False)
one_hot_label = tf.one_hot(label, depth=NUM_CLASSES, axis=1, dtype=tf.float32)
masked_v = tf.matmul(tf.squeeze(caps),
tf.reshape(one_hot_label, (-1, NUM_CLASSES, 1)), transpose_a=True)
with tf.variable_scope('Reconstruct'):
v_j = tf.reshape(masked_v, shape=(batch_size, -1))
fc = tf.contrib.layers.fully_connected(inputs=v_j, num_outputs=128, scope="fc1", biases_initializer=None)
fc = slim.batch_norm(fc,
center=True,
scale=True,
is_training=is_training,
scope="fc1", fused=False, decay=bn_decay)
fc = tf.contrib.layers.fully_connected(inputs=fc, num_outputs=256, scope="fc2", biases_initializer=None)
fc = slim.batch_norm(fc,
center=True,
scale=True,
is_training=is_training,
scope="fc2", fused=False, decay=bn_decay)
fc = tf.contrib.layers.fully_connected(inputs=fc, num_outputs=512, scope="fc3", biases_initializer=None)
fc = slim.batch_norm(fc,
center=True,
scale=True,
is_training=is_training,
scope="fc3", fused=False, decay=bn_decay)
fc = tf.contrib.layers.fully_connected(inputs=fc, num_outputs=1024*3, scope="out", biases_initializer=None, activation_fn=None)
net = tf.reshape(fc, (batch_size, 1024, 1, 3))
reconstruct = tf.squeeze(net)
end_points['reconstruct'] = tf.reshape(reconstruct, shape=(batch_size, 1024, 3))
return caps, end_points
def get_instance_seg_v2_net(point_cloud, feature_vec, cls_label, is_training,
bn_decay, end_points):
''' 3D instance segmentation PointNet v2 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
feature_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 1])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
32, [0.4, 0.8, 1.6], [64, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[128, 256, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# classification
batch_size = tf.shape(feature_vec)[0]
cls_net = tf.reshape(l3_points, [batch_size, -1])
cls_net = tf.concat([cls_net, feature_vec], axis=1)
cls_net = tf_util.fully_connected(cls_net,
512,
bn=True,
is_training=is_training,
scope='cls_fc1',
bn_decay=bn_decay)
#cls_net = tf_util.dropout(cls_net, keep_prob=0.4, is_training=is_training, scope='cls_dp1')
cls_net = tf_util.fully_connected(cls_net,
256,
bn=True,
is_training=is_training,
scope='cls_fc2',
bn_decay=bn_decay)
#cls_net = tf_util.dropout(cls_net, keep_prob=0.4, is_training=is_training, scope='cls_dp2')
cls_net = tf_util.fully_connected(cls_net,
NUM_OBJ_CLASSES,
activation_fn=None,
scope='cls_logits')
end_points['cls_logits'] = cls_net
cls_label_pred = tf.argmax(tf.nn.softmax(end_points['cls_logits']), axis=1)
end_points['one_hot_vec'] = tf.one_hot(cls_label_pred, NUM_OBJ_CLASSES)
end_points['one_hot_gt'] = tf.one_hot(cls_label, NUM_OBJ_CLASSES)
# Feature Propagation layers
one_hot_vec = tf.cond(is_training, lambda: end_points['one_hot_gt'],
lambda: end_points['one_hot_vec'])
l3_points = tf.concat([l3_points, tf.expand_dims(one_hot_vec, 1)], axis=2)
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer2')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat([l0_xyz, l0_points], axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='conv1d-fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope='dp1')
logits = tf_util.conv1d(net,
2,
1,
padding='VALID',
activation_fn=None,
scope='conv1d-fc2')
return logits, end_points
def trans_pred_net(xyz,
flow,
scopename,
reuse,
is_training,
bn_decay=None,
nfea=12):
#########################
# input
# xyz: (B x N x 3)
# flow: (B x N x 3)
# output
# pred_trans: (B x N x nfea)
#########################
num_point = xyz.get_shape()[1].value
with tf.variable_scope(scopename) as myscope:
if reuse:
myscope.reuse_variables()
l0_xyz = xyz
l0_points = flow
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module_msg(
l0_xyz,
l0_points,
256, [0.1, 0.2], [64, 64], [[64, 64], [64, 64], [64, 128]],
is_training,
bn_decay,
scope='trans_layer1',
centralize_points=True)
l2_xyz, l2_points, l2_indices = pointnet_sa_module(
l1_xyz,
l1_points,
npoint=128,
radius=0.4,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='trans_layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(
l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
use_xyz=True,
is_training=is_training,
bn_decay=bn_decay,
scope='trans_layer3')
# Feature Propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='trans_fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='trans_fa_layer2')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 64],
is_training,
bn_decay,
scope='trans_fa_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
64,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='trans_fc1',
bn_decay=bn_decay)
net = tf_util.conv1d(net,
nfea,
1,
padding='VALID',
activation_fn=None,
scope='trans_fc2')
pred_trans = tf.reshape(net, [-1, num_point, nfea])
return pred_trans
def get_model_w_ae_p(self, point_cloud, is_training, bn_decay=None):
"""" Classification PointNet, input is BxNx3, output Bx40 """
pointnet_util = imp.load_source(
'pointnet_util',
os.path.join(os.path.dirname(self.models["test"]), '../utils',
"pointnet_util.py"))
tf_util = imp.load_source(
'tf_util',
os.path.join(os.path.dirname(self.models["test"]), '../utils',
"tf_util.py"))
from pointnet_util import pointnet_sa_module, pointnet_sa_module_msg
batch_size = self.configuration.batch_size
num_point = self.configuration.n_input[0]
end_points = {}
l0_xyz = point_cloud
l0_points = None
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1',
use_nchw=True)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
end_points['post_max'] = net
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp2')
end_points['final'] = net
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label,
depth=NUM_CATEGORIES,
on_value=1.0,
off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot,
[batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat(
[cls_label_one_hot, l0_xyz, l0_points],
axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
end_points['feats'] = net
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
k = 4
# Set abstraction layers
_, tl0_points = pointnet_triple_module(l0_xyz,
l0_points, [32, 32],
is_training,
bn_decay,
scope='tlayer1',
knn=k)
l0_points = tf.concat((l0_points, tl0_points), axis=-1)
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1',
use_nchw=True)
_, tl1_points = pointnet_triple_module(l1_xyz,
l1_points, [64, 64],
is_training,
bn_decay,
scope='tlayer2',
knn=k)
l1_points = tf.concat((l1_points, tl1_points), axis=-1)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2')
_, tl2_points = pointnet_triple_module(l2_xyz,
l2_points, [128, 128],
is_training,
bn_decay,
scope='tlayer3',
knn=k)
l2_points = tf.concat((l2_points, tl2_points), axis=-1)
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.4,
is_training=is_training,
scope='dp2')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_model(point_cloud,
is_training,
n_classes,
bn_decay=None,
weight_decay=None,
inception=True,
**kwargs):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
if point_cloud.shape[2] > 3:
l0_xyz = point_cloud[:, :, :3]
l0_points = point_cloud[:, :, 3:]
else:
l0_xyz = point_cloud
l0_points = None
end_points['l0_xyz'] = l0_xyz
input_image = tf.expand_dims(point_cloud, 2)
net, conv_ker = tf_util.inception(input_image,
64,
scope='seq_conv1',
kernel_heights=[1, 3, 5, 7],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 2, 2, 2],
return_kernel=True,
bn=True,
bn_decay=bn_decay,
is_training=is_training)
end_points['conv_ker'] = conv_ker
net, conv_ker = tf_util.inception(input_image,
32,
scope='seq_conv2',
kernel_heights=[1, 3, 5, 7],
kernel_widths=[1, 1, 1, 1],
kernels_fraction=[2, 2, 2, 2],
return_kernel=True,
bn=True,
bn_decay=bn_decay,
is_training=is_training)
conv_net = tf.squeeze(net)
l0_points = tf.concat([conv_net, l0_points], axis=-1)
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [16, 32, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1',
use_nchw=True,
weight_decay=weight_decay)
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.2, 0.4, 0.8], [32, 64, 128],
[[64, 64, 128], [128, 128, 256], [128, 128, 256]],
is_training,
bn_decay,
scope='layer2',
weight_decay=weight_decay)
l3_xyz, l3_points, _, _ = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3',
weight_decay=weight_decay)
# Fully connected layers
net = tf.reshape(l3_points, [batch_size, -1])
net = tf_util.fully_connected(net,
512,
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay,
weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp1')
net = tf_util.fully_connected(net,
256,
bn=True,
is_training=is_training,
scope='fc2',
bn_decay=bn_decay,
weight_decay=weight_decay)
net = tf_util.dropout(net, rate=0.6, is_training=is_training, scope='dp2')
net = tf_util.fully_connected(net,
n_classes,
activation_fn=None,
scope='fc3')
return net, end_points
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
farthest_distance = 0.15
num_neighbors = 4
#######Contextual representation
new_xyz = l0_xyz # (batch_size, npoint, 3)
idx, pts_cnt = query_ball_point(farthest_distance, num_neighbors, l0_xyz,
new_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
neighbor_xyz -= tf.tile(tf.expand_dims(new_xyz, 2),
[1, 1, num_neighbors, 1])
neighbor_points = group_point(l0_points, idx)
neighbor_representation = tf.concat([neighbor_xyz, neighbor_points],
axis=-1)
neighbor_representation = tf.reshape(neighbor_representation,
(batch_size, num_point, -1))
num_channel = neighbor_representation.get_shape()[2].value
points = tf_util.conv1d(point_cloud,
num_channel,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='points_fc',
bn_decay=bn_decay)
neighbor_representation_gp = gating_process(
neighbor_representation,
num_channel,
padding='VALID',
is_training=is_training,
scope='neighbor_representation_gp',
bn_decay=bn_decay)
points_gp = gating_process(points,
num_channel,
padding='VALID',
is_training=is_training,
scope='points_gp',
bn_decay=bn_decay)
l0_points_CR = tf.concat([
neighbor_representation_gp * points,
points_gp * neighbor_representation
],
axis=-1)
l0_points = l0_points_CR
########## Positional Representation
#num_channel=K
idx, pts_cnt = query_ball_point(0.2, 16, l0_xyz, l0_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
# neighbor_xyz = self.gather_neighbour(xyz, neigh_idx)
xyz_tile = tf.tile(tf.expand_dims(l0_xyz, axis=2),
[1, 1, tf.shape(idx)[-1], 1])
relative_xyz = xyz_tile - neighbor_xyz
#relative_xyz =neighbor_xyz
relative_dis = tf.reduce_sum(tf.square(relative_xyz),
axis=-1,
keepdims=True)
encoded_position = tf.concat(
[relative_dis, relative_xyz, xyz_tile, neighbor_xyz], axis=-1)
encoded_position = tf_util.conv2d(encoded_position,
16, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv011',
bn_decay=bn_decay)
encoded_neighbours = group_point(l0_points, idx)
positional_representation = tf.concat(
[encoded_neighbours, encoded_position], axis=-1)
positional_representation = tf.reduce_mean(positional_representation,
axis=[2],
keep_dims=True,
name='avgpool')
points = tf_util.conv2d(tf.concat(
[positional_representation,
tf.expand_dims(l0_points_CR, 2)], axis=-1),
num_channel * 2, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='attp',
bn_decay=bn_decay)
points = tf.squeeze(points, [2])
end_points['points'] = points
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label,
depth=NUM_CATEGORIES,
on_value=1.0,
off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot,
[batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat(
[cls_label_one_hot, l0_xyz, l0_points],
axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
########## Channel-wise Attention
input = net
output_a = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_a',
bn_decay=bn_decay)
output_b = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_b',
bn_decay=bn_decay)
output_b = tf.transpose(output_b, [0, 1, 3, 2])
output_a = tf.squeeze(output_a, [1])
output_b = tf.squeeze(output_b, [1])
energy = tf.matmul(output_b, output_a)
D = tf.reduce_max(energy, -1)
D = tf.expand_dims(D, -1)
energy_new = tf.tile(D, multiples=[1, 1, energy.shape[2]]) - energy
attention = tf.nn.softmax(energy_new, axis=-1)
output_d = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_d',
bn_decay=bn_decay)
output_d = tf.squeeze(output_d, [1])
output_CA = tf.matmul(output_d, attention)
gamma2 = tf_util._variable_with_weight_decay('weightsgamma2m',
shape=[1],
use_xavier=True,
stddev=1e-3,
wd=0.0)
output_CA = output_CA * gamma2 + input
########## Squeeze-and-Excitation
ex1 = tf.reduce_mean(input, axis=[1], keep_dims=True, name='avgpool1')
print(ex1.get_shape())
ex1 = tf_util.conv1d(ex1,
64,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='ex1',
bn_decay=bn_decay)
print(ex1.get_shape())
ex1 = tf_util.conv1d(ex1,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='ex2',
bn_decay=bn_decay)
print(ex1.get_shape())
output2 = input * ex1
output = tf.concat([output_CA, output2], axis=-1)
end_points['feats'] = output
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
