def get_model(point_cloud,
one_hot_vec,
is_training,
bn_decay=None,
track=False,
lstm_params=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
track : true to use track ids for the amodal box estimation pointnet
lstm_params: A dictionary that contains lstm parameters; n_batch, tau, feat_vec_len
This is only used if track is True.
Output:
end_points: dict (map from name strings to TF tensors)
'''
end_points = {}
# 3D Instance Segmentation PointNet
logits, end_points = get_instance_seg_v1_net(point_cloud, one_hot_vec,
is_training, bn_decay,
end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = point_cloud_masking(
point_cloud, logits, end_points)
# T-Net and coordinate translation
center_delta, end_points = get_center_regression_net(
object_point_cloud_xyz, one_hot_vec, is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = object_point_cloud_xyz - tf.expand_dims(
center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points = get_3d_box_estimation_v1_net(
object_point_cloud_xyz_new,
one_hot_vec,
is_training,
bn_decay,
end_points,
track=track,
lstm_params=lstm_params,
center_est=stage1_center)
# Parse output to 3D box parameters
end_points = parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
end_points = {}
# 3D Instance Segmentation PointNet
logits, end_points = get_instance_seg_v1_net(\
point_cloud, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = \
point_cloud_masking(point_cloud, logits, end_points)
end_points['object_point_cloud_xyz'] = object_point_cloud_xyz
end_points['mask_xyz_mean'] = mask_xyz_mean
# T-Net and coordinate translation
center_delta, end_points = get_center_regression_net(\
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
end_points['center_delta'] = center_delta
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = \
object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points = get_3d_box_estimation_v1_net(\
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# Parse output to 3D box parameters
end_points = parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
end_points = {}
# 3D Instance Segmentation PointNet
logits, end_points = get_instance_seg_v1_net(\
point_cloud, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = \
point_cloud_masking(point_cloud, logits, end_points)
# T-Net and coordinate translation
center_delta, end_points = get_center_regression_net(\
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = \
object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points = get_3d_box_estimation_v1_net(\
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# Parse output to 3D box parameters
end_points = parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
end_points = {}
# 3D实例分割PointNet
logits, end_points = get_instance_seg_v1_net(
point_cloud, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# 根据mask筛选点云，并转换到mask点云中心
object_point_cloud_xyz, mask_xyz_mean, end_points = \
point_cloud_masking(point_cloud, logits, end_points)
# T-Net中心残差回归网络和坐标转换
center_delta, end_points = get_center_regression_net(
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center # 物体坐标系中心在原坐标下的位置
# 获得物体坐标系下的物体点云
object_point_cloud_xyz_new = object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# 边框估计PointNet
output, end_points = get_3d_box_estimation_v1_net(
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# 将输出拆分并进行保存
end_points = parse_output_to_tensors(output, end_points)
# center_boxnet为3D边框估计网络估计的边框中心，加上物体坐标系中心在原坐标下的位置，得到边框中心在原坐标系下的位置
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
#############Invariance transformation Net###########################################
### Add Neighboring feature
### generate new only xyz coordinate point cloud tensor -- no intensity
point_cloud_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
print("point_cloud shape", point_cloud.get_shape())
print("point_cloud_xyz", point_cloud_xyz.get_shape())
features_initial = None
invariance_transformation_net = Invariance_Transformation_Net(
point_cloud=point_cloud_xyz,
features=features_initial,
is_training=is_training,
invarians_trans_param=invariants_trans_param_7_layer).layer_fts[-1]
print("invariance_transformation_net",
tf.shape(invariance_transformation_net))
print("invariance_transformation_net", type(invariance_transformation_net))
print(
'----------------------------------done----------------------------------------------'
)
end_points = {}
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
k = 4 ###################Set the number of neighboring #################################################################################################
adj_matrix = pairwise_distance(point_cloud_xyz)
print("adj_matrix", adj_matrix.get_shape())
nn_idx = knn(adj_matrix, k=k)
print("nn_idx", nn_idx.get_shape())
#edge_feature=get_edge_feature(point_cloud,point_cloud_xyz,nn_idx=nn_idx,k=k)
edge_feature = get_edge_feature(point_cloud_xyz, nn_idx=nn_idx, k=k)
print("edge_feature", edge_feature.get_shape())
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3)
poinr_cloud_transformed = tf.matmul(point_cloud_xyz, transform)
print("edge_transform_feature", poinr_cloud_transformed.get_shape())
point_cloud_concat = tf.concat([point_cloud, poinr_cloud_transformed],
axis=-1)
print("point_cloud_concat", point_cloud_concat.get_shape())
####################################Invariance transformation features added##############################################################################
point_cloud_invari = tf.concat(
[point_cloud_concat, invariance_transformation_net], axis=-1)
#point_cloud_invari= tf.concat([point_cloud,invariance_transformation_net],axis=-1)
print("point_cloud_invari", point_cloud_invari.get_shape())
# 3D Instance Segmentation PointNet
#logits, end_points = get_instance_seg_v1_net(\
# point_cloud_concat, one_hot_vec,
# is_training, bn_decay, end_points)
#end_points['mask_logits'] = logits
logits, end_points = get_instance_seg_v1_net(\
point_cloud_invari, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = \
point_cloud_masking(point_cloud, logits, end_points)
# T-Net and coordinate translation
center_delta, end_points = get_center_regression_net(\
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = \
object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points = get_3d_box_estimation_v1_net(\
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# Parse output to 3D box parameters
end_points = parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
#############Invariance transformation Net###########################################
### Add Neighboring feature
### generate new only xyz coordinate point cloud tensor -- no intensity
point_cloud_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
print("point_cloud shape", point_cloud.get_shape())
print("point_cloud_xyz", point_cloud_xyz.get_shape())
end_points = {}
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
k = 2 ###################Set the number of neighboring #################################################################################################
adj_matrix = pairwise_distance(point_cloud_xyz)
print("adj_matrix", adj_matrix.get_shape())
nn_idx = knn(adj_matrix, k=k)
print("nn_idx", nn_idx.get_shape())
#edge_feature=get_edge_feature(point_cloud,point_cloud_xyz,nn_idx=nn_idx,k=k)
edge_feature = get_edge_feature(point_cloud_xyz, nn_idx=nn_idx, k=k)
print("edge_feature", edge_feature.get_shape())
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(edge_feature,
is_training,
bn_decay,
K=3)
poinr_cloud_transformed = tf.matmul(point_cloud_xyz, transform)
print("edge_transform_feature", poinr_cloud_transformed.get_shape())
adj_matrix_edge = pairwise_distance(poinr_cloud_transformed)
print("adj_matrix_edg_0", adj_matrix_edge.get_shape())
nn_idx = knn(adj_matrix_edge, k=k)
print("nn_idx_0", nn_idx.get_shape())
edge_feature_edge = get_edge_feature(poinr_cloud_transformed,
nn_idx=nn_idx,
k=k)
print("edge_feature_edge_0", edge_feature_edge.get_shape())
edge_net = tf_util.conv2d_dgcnn(edge_feature_edge,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
bn_decay=bn_decay,
scope="edge_conv_0")
print("edge_feature_conv_0", edge_net.get_shape())
edge_net = tf.reduce_max(edge_net, axis=-2, keep_dims=True)
print("edge_net_change_channel_0", edge_net.get_shape())
net1 = edge_net
adj_matrix_edge = pairwise_distance(edge_net)
print("adj_matrix_edg_1", adj_matrix_edge.get_shape())
nn_idx = knn(adj_matrix_edge, k=k)
edge_net = get_edge_feature(edge_net, nn_idx=nn_idx, k=k)
edge_net = tf_util.conv2d_dgcnn(edge_net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
bn_decay=bn_decay,
scope="edge_conv_1")
edge_net = tf.reduce_max(edge_net, axis=-2, keep_dims=True)
print("edge_net_change_channel_1", edge_net.get_shape())
net2 = edge_net
adj_matrix_edge = pairwise_distance(edge_net)
print("adj_matrix_edg_2", adj_matrix_edge.get_shape())
nn_idx = knn(adj_matrix_edge, k=k)
edge_net = get_edge_feature(edge_net, nn_idx=nn_idx, k=k)
edge_net = tf_util.conv2d_dgcnn(edge_net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
bn_decay=bn_decay,
scope="edge_conv_2")
edge_net = tf.reduce_max(edge_net, axis=-2, keep_dims=True)
print("edge_net_change_channel_2", edge_net.get_shape())
net3 = edge_net
print("net3", net3.get_shape())
net4 = tf.squeeze(net3, axis=-2)
point_cloud_concat = tf.concat([point_cloud, net4], axis=-1)
print("point_cloud_concat", point_cloud_concat.get_shape())
logits, end_points = get_instance_seg_v1_net(\
point_cloud_concat, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = \
point_cloud_masking(point_cloud, logits, end_points)
# T-Net and coordinate translation
center_delta, end_points = get_center_regression_net(\
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = \
object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points = get_3d_box_estimation_v1_net(\
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# Parse output to 3D box parameters
end_points = parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points
