def __init__(self, k=20): super(classification_net, self).__init__() self.conv1 = nn.Conv2d(6, 64, kernel_size=1) self.conv2 = nn.Conv2d(128, 64, kernel_size=1) self.conv3 = nn.Conv2d(128, 64, kernel_size=1) self.conv4 = nn.Conv2d(128, 128, kernel_size=1) self.conv5 = nn.Conv2d(320, 1024, kernel_size=1) self.bn1 = nn.BatchNorm2d(64) self.bn2 = nn.BatchNorm2d(64) self.bn3 = nn.BatchNorm2d(64) self.bn4 = nn.BatchNorm2d(128) self.bn5 = nn.BatchNorm2d(1024) self.bn6 = nn.BatchNorm1d(512) self.bn7 = nn.BatchNorm1d(256) self.fc1 = nn.Linear(1024, 512) self.fc2 = nn.Linear(512, 256) self.fc3 = nn.Linear(256, 40) self.dropout = nn.Dropout(p=0.5) self.input_transform = input_transform_net() self.k = k
def __init__(self, part_num, k=30, cat_num=16):
super(part_seg_net, self).__init__()
self.conv1 = nn.Conv2d(6, 64, kernel_size=1)
self.conv2 = nn.Conv2d(64, 64, kernel_size=1)
self.conv3 = nn.Conv2d(128, 64, kernel_size=1)
self.conv4 = nn.Conv2d(128, 64, kernel_size=1)
self.conv5 = nn.Conv2d(128, 64, kernel_size=1)
self.conv6 = nn.Conv2d(128, 64, kernel_size=1)
self.conv7 = nn.Conv2d(128, 64, kernel_size=1)
self.conv8 = nn.Conv2d(192, 1024, kernel_size=1)
self.conv9 = nn.Conv2d(cat_num, 128, kernel_size=1)
self.conv10 = nn.Conv2d(2752, 256, kernel_size=1)
self.conv11 = nn.Conv2d(256, 256, kernel_size=1)
self.conv12 = nn.Conv2d(256, 128, kernel_size=1)
self.conv13 = nn.Conv2d(128, part_num, kernel_size=1)
self.bn1 = nn.BatchNorm2d(64)
self.bn2 = nn.BatchNorm2d(64)
self.bn3 = nn.BatchNorm2d(64)
self.bn4 = nn.BatchNorm2d(64)
self.bn5 = nn.BatchNorm2d(64)
self.bn6 = nn.BatchNorm2d(64)
self.bn7 = nn.BatchNorm2d(64)
self.bn8 = nn.BatchNorm2d(1024)
self.bn9 = nn.BatchNorm2d(128)
self.bn10 = nn.BatchNorm2d(256)
self.bn11 = nn.BatchNorm2d(256)
self.bn12 = nn.BatchNorm2d(128)
self.dropout = nn.Dropout(p=0.6)
self.input_transform = input_transform_net()
self.k = k
self.part_num = part_num
self.cat_num = cat_num
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
#############Invariance transformation Net###########################################
### Add Neighboring feature
### generate new only xyz coordinate point cloud tensor -- no intensity
point_cloud_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
print("point_cloud shape", point_cloud.get_shape())
print("point_cloud_xyz", point_cloud_xyz.get_shape())
features_initial = None
invariance_transformation_net = Invariance_Transformation_Net(
point_cloud=point_cloud_xyz,
features=features_initial,
is_training=is_training,
invarians_trans_param=invariants_trans_param_7_layer).layer_fts[-1]
print("invariance_transformation_net",
tf.shape(invariance_transformation_net))
print("invariance_transformation_net", type(invariance_transformation_net))
print(
'----------------------------------done----------------------------------------------'
)
end_points = {}
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
k = 4 ###################Set the number of neighboring #################################################################################################
adj_matrix = pairwise_distance(point_cloud_xyz)
print("adj_matrix", adj_matrix.get_shape())
nn_idx = knn(adj_matrix, k=k)
print("nn_idx", nn_idx.get_shape())
#edge_feature=get_edge_feature(point_cloud,point_cloud_xyz,nn_idx=nn_idx,k=k)
edge_feature = get_edge_feature(point_cloud_xyz, nn_idx=nn_idx, k=k)
print("edge_feature", edge_feature.get_shape())
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3)
poinr_cloud_transformed = tf.matmul(point_cloud_xyz, transform)
print("edge_transform_feature", poinr_cloud_transformed.get_shape())
point_cloud_concat = tf.concat([point_cloud, poinr_cloud_transformed],
axis=-1)
print("point_cloud_concat", point_cloud_concat.get_shape())
####################################Invariance transformation features added##############################################################################
point_cloud_invari = tf.concat(
[point_cloud_concat, invariance_transformation_net], axis=-1)
#point_cloud_invari= tf.concat([point_cloud,invariance_transformation_net],axis=-1)
print("point_cloud_invari", point_cloud_invari.get_shape())
# 3D Instance Segmentation PointNet
#logits, end_points = get_instance_seg_v1_net(\
# point_cloud_concat, one_hot_vec,
# is_training, bn_decay, end_points)
#end_points['mask_logits'] = logits
logits, end_points = get_instance_seg_v1_net(\
point_cloud_invari, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = \
point_cloud_masking(point_cloud, logits, end_points)
# T-Net and coordinate translation
center_delta, end_points = get_center_regression_net(\
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = \
object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points = get_3d_box_estimation_v1_net(\
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# Parse output to 3D box parameters
end_points = parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
#############Invariance transformation Net###########################################
### Add Neighboring feature
### generate new only xyz coordinate point cloud tensor -- no intensity
point_cloud_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
print("point_cloud shape", point_cloud.get_shape())
print("point_cloud_xyz", point_cloud_xyz.get_shape())
end_points = {}
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
k = 2 ###################Set the number of neighboring #################################################################################################
adj_matrix = pairwise_distance(point_cloud_xyz)
print("adj_matrix", adj_matrix.get_shape())
nn_idx = knn(adj_matrix, k=k)
print("nn_idx", nn_idx.get_shape())
#edge_feature=get_edge_feature(point_cloud,point_cloud_xyz,nn_idx=nn_idx,k=k)
edge_feature = get_edge_feature(point_cloud_xyz, nn_idx=nn_idx, k=k)
print("edge_feature", edge_feature.get_shape())
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3)
poinr_cloud_transformed = tf.matmul(point_cloud_xyz, transform)
print("edge_transform_feature", poinr_cloud_transformed.get_shape())
adj_matrix_edge = pairwise_distance(poinr_cloud_transformed)
print("adj_matrix_edg_0", adj_matrix_edge.get_shape())
nn_idx = knn(adj_matrix_edge, k=k)
print("nn_idx_0", nn_idx.get_shape())
edge_feature_edge = get_edge_feature(poinr_cloud_transformed,
nn_idx=nn_idx,
k=k)
print("edge_feature_edge_0", edge_feature_edge.get_shape())
edge_net = tf_util.conv2d_dgcnn(edge_feature_edge,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
bn_decay=bn_decay,
scope="edge_conv_0")
print("edge_feature_conv_0", edge_net.get_shape())
edge_net = tf.reduce_max(edge_net, axis=-2, keep_dims=True)
print("edge_net_change_channel_0", edge_net.get_shape())
net1 = edge_net
adj_matrix_edge = pairwise_distance(edge_net)
print("adj_matrix_edg_1", adj_matrix_edge.get_shape())
nn_idx = knn(adj_matrix_edge, k=k)
edge_net = get_edge_feature(edge_net, nn_idx=nn_idx, k=k)
edge_net = tf_util.conv2d_dgcnn(edge_net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
bn_decay=bn_decay,
scope="edge_conv_1")
edge_net = tf.reduce_max(edge_net, axis=-2, keep_dims=True)
print("edge_net_change_channel_1", edge_net.get_shape())
net2 = edge_net
adj_matrix_edge = pairwise_distance(edge_net)
print("adj_matrix_edg_2", adj_matrix_edge.get_shape())
nn_idx = knn(adj_matrix_edge, k=k)
edge_net = get_edge_feature(edge_net, nn_idx=nn_idx, k=k)
edge_net = tf_util.conv2d_dgcnn(edge_net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
bn_decay=bn_decay,
scope="edge_conv_2")
edge_net = tf.reduce_max(edge_net, axis=-2, keep_dims=True)
print("edge_net_change_channel_2", edge_net.get_shape())
net3 = edge_net
print("net3", net3.get_shape())
net4 = tf.squeeze(net3, axis=-2)
point_cloud_concat = tf.concat([point_cloud, net4], axis=-1)
print("point_cloud_concat", point_cloud_concat.get_shape())
logits, end_points = get_instance_seg_v1_net(\
point_cloud_concat, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = \
point_cloud_masking(point_cloud, logits, end_points)
# T-Net and coordinate translation
center_delta, end_points = get_center_regression_net(\
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = \
object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points = get_3d_box_estimation_v1_net(\
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# Parse output to 3D box parameters
end_points = parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points