def test_sparse_voxel_grid_to_pointcloud(self):
voxel_features_0 = tf.constant(
[[0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0],
[0.0, 0.0, 0.0], [1.0, 1.0, 1.0]],
dtype=tf.float32)
voxel_features_1 = tf.constant(
[[0.0, 0.0, 0.5], [0.0, 0.5, 0.0], [0.5, 0.0, 0.0],
[0.0, 0.0, 0.0], [0.5, 0.5, 0.5]],
dtype=tf.float32)
voxel_features = tf.stack([voxel_features_0, voxel_features_1], axis=0)
segment_ids = tf.constant(
[[0, 0, 1, 1, 2, 2, 0, 0, 0, 0], [1, 3, 1, 2, 0, 4, 4, 0, 0, 0]],
dtype=tf.int32)
num_valid_voxels = tf.constant([3, 5], dtype=tf.int32)
num_valid_points = tf.constant([7, 9], dtype=tf.int32)
point_features = voxel_utils.sparse_voxel_grid_to_pointcloud(
voxel_features=voxel_features,
segment_ids=segment_ids,
num_valid_voxels=num_valid_voxels,
num_valid_points=num_valid_points)
np_point_features = point_features.numpy()
self.assertAllEqual(np_point_features.shape, [2, 10, 3])
self.assertAllClose(
np_point_features[0],
np.array([[0.0, 0.0, 1.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0],
[0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0],
[0.0, 0.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0],
[0.0, 0.0, 0.0]]))
self.assertAllClose(
np_point_features[1],
np.array([[0.0, 0.5, 0.0], [0.0, 0.0, 0.0], [0.0, 0.5, 0.0],
[0.5, 0.0, 0.0], [0.0, 0.0, 0.5], [0.5, 0.5, 0.5],
[0.5, 0.5, 0.5], [0.0, 0.0, 0.5], [0.0, 0.0, 0.5],
[0.0, 0.0, 0.0]]))
def __call__(self, inputs, training=True):
"""Runs the model and returns the semantic logits prediction.
Args:
inputs: A dictionary of tensors containing inputs for the model.
training: Whether the model runs in training mode.
Returns:
A dictionary of tensors containing semantic logits prediction.
"""
inputs[standard_fields.InputDataFields.num_valid_voxels] = tf.reshape(
inputs[standard_fields.InputDataFields.num_valid_voxels], [-1])
voxel_inputs = (
inputs[standard_fields.InputDataFields.voxel_features],
inputs[standard_fields.InputDataFields.voxel_xyz_indices],
inputs[standard_fields.InputDataFields.num_valid_voxels])
outputs = self.sparse_conv_unet(voxel_inputs, training=training)
# If at eval time, transfer voxel features to points
if ((not training)
and (standard_fields.InputDataFields.points_to_voxel_mapping
in inputs)):
voxel_to_point_mapping = (
standard_fields.get_output_voxel_to_point_field_mapping())
point_tensor_outputs = {}
for task_name in outputs:
if task_name in voxel_to_point_mapping and outputs[
task_name] is not None:
point_tensor_outputs[voxel_to_point_mapping[task_name]] = (
voxel_utils.sparse_voxel_grid_to_pointcloud(
voxel_features=outputs[task_name],
segment_ids=inputs[standard_fields.InputDataFields.
points_to_voxel_mapping],
num_valid_voxels=inputs[
standard_fields.InputDataFields.
num_valid_voxels],
num_valid_points=inputs[
standard_fields.InputDataFields.
num_valid_points]))
outputs.update(point_tensor_outputs)
model_utils.postprocess(
inputs=inputs,
outputs=outputs,
is_training=training,
num_furthest_voxel_samples=self.num_furthest_voxel_samples,
sampler_score_vs_distance_coef=self.sampler_score_vs_distance_coef,
embedding_similarity_strategy=self.embedding_similarity_strategy,
embedding_similarity_threshold=self.embedding_similarity_threshold,
score_threshold=self.nms_score_threshold,
apply_nms=self.apply_nms,
nms_iou_threshold=self.nms_iou_threshold)
if training:
self.calculate_losses(inputs=inputs, outputs=outputs)
return outputs
def call(self, inputs, training=True):
"""Runs the model and returns the semantic logits prediction.
Args:
inputs: A dictionary of tensors containing inputs for the model.
training: Whether the model runs in training mode.
Returns:
A dictionary of tensors containing semantic logits prediction.
"""
# when not using custom train step, this field becomes 2 dim.
inputs[standard_fields.InputDataFields.num_valid_voxels] = tf.reshape(
inputs[standard_fields.InputDataFields.num_valid_voxels], [-1])
voxel_inputs = (
inputs[standard_fields.InputDataFields.voxel_features],
inputs[standard_fields.InputDataFields.voxel_xyz_indices],
inputs[standard_fields.InputDataFields.num_valid_voxels])
outputs = self.sparse_conv_unet(voxel_inputs, training=training)
# If at eval time, transfer voxel features to points
if ((not training)
and (standard_fields.InputDataFields.points_to_voxel_mapping
in inputs)):
inputs[
standard_fields.InputDataFields.num_valid_points] = tf.reshape(
inputs[standard_fields.InputDataFields.num_valid_points],
[-1])
voxel_to_point_mapping = (
standard_fields.get_output_voxel_to_point_field_mapping())
point_tensor_outputs = {}
for task_name in outputs:
if task_name in voxel_to_point_mapping and outputs[
task_name] is not None:
point_tensor_outputs[voxel_to_point_mapping[task_name]] = (
voxel_utils.sparse_voxel_grid_to_pointcloud(
voxel_features=outputs[task_name],
segment_ids=inputs[standard_fields.InputDataFields.
points_to_voxel_mapping],
num_valid_voxels=inputs[
standard_fields.InputDataFields.
num_valid_voxels],
num_valid_points=inputs[
standard_fields.InputDataFields.
num_valid_points]))
# include fields used by tensorboard visualization call back.
outputs.update(point_tensor_outputs)
if training:
self.calculate_losses(inputs=inputs, outputs=outputs)
return outputs
def __call__(self, inputs, training=True):
"""Runs the model and returns the semantic logits prediciton.
Args:
inputs: A dictionary of tensors containing inputs for the model.
training: Whether the model runs in training mode.
Returns:
A dictionary of tensors containing semantic logits prediciton.
"""
# when not using custom train step, this field becomes 2 dim.
inputs[standard_fields.InputDataFields.num_valid_voxels] = tf.reshape(
inputs[standard_fields.InputDataFields.num_valid_voxels], [-1])
voxel_inputs = (
inputs[standard_fields.InputDataFields.voxel_features],
inputs[standard_fields.InputDataFields.voxel_xyz_indices],
inputs[standard_fields.InputDataFields.num_valid_voxels])
outputs = self.sparse_conv_hourglass(voxel_inputs, training=training)
outputs[standard_fields.DetectionResultFields.
object_center_voxels] += inputs[
standard_fields.InputDataFields.voxel_positions]
model_utils.rectify_outputs(outputs=outputs)
# If at eval time, transfer voxel features to points
if ((not training)
and (standard_fields.InputDataFields.points_to_voxel_mapping
in inputs)):
inputs[
standard_fields.InputDataFields.num_valid_points] = tf.reshape(
inputs[standard_fields.InputDataFields.num_valid_points],
[-1])
voxel_to_point_mapping = (
standard_fields.get_output_voxel_to_point_field_mapping())
point_tensor_outputs = {}
for task_name in outputs:
if task_name in voxel_to_point_mapping and outputs[
task_name] is not None:
point_tensor_outputs[voxel_to_point_mapping[task_name]] = (
voxel_utils.sparse_voxel_grid_to_pointcloud(
voxel_features=outputs[task_name],
segment_ids=inputs[standard_fields.InputDataFields.
points_to_voxel_mapping],
num_valid_voxels=inputs[
standard_fields.InputDataFields.
num_valid_voxels],
num_valid_points=inputs[
standard_fields.InputDataFields.
num_valid_points]))
outputs.update(point_tensor_outputs)
model_utils.postprocess(
inputs=inputs,
outputs=outputs,
is_training=training,
apply_nms=self.apply_nms,
nms_score_threshold=self.nms_score_threshold,
nms_iou_threshold=self.nms_iou_threshold,
nms_max_num_predicted_boxes=self.nms_max_num_predicted_boxes,
use_furthest_voxel_sampling=self.use_furthest_voxel_sampling,
num_furthest_voxel_samples=self.num_furthest_voxel_samples,
sampler_score_vs_distance_coef=self.sampler_score_vs_distance_coef)
if training:
self.calculate_losses(inputs=inputs, outputs=outputs)
return outputs
def postprocess(inputs, outputs, is_training, num_furthest_voxel_samples,
sampler_score_vs_distance_coef, embedding_similarity_strategy,
embedding_similarity_threshold, score_threshold, apply_nms,
nms_iou_threshold):
"""Post-processor function.
Args:
inputs: A dictionary containing input tensors.
outputs: A dictionary containing predicted tensors.
is_training: If during training stage or not.
num_furthest_voxel_samples: Number of voxels to be sampled using furthest
voxel sampling in the postprocessor.
sampler_score_vs_distance_coef: The coefficient that balances the weight
between furthest voxel sampling and highest score sampling in the
postprocessor.
embedding_similarity_strategy: Embedding similarity strategy.
embedding_similarity_threshold: Similarity threshold used to decide if two
point embedding vectors belong to the same instance.
score_threshold: Instance score threshold used throughout postprocessing.
apply_nms: If True, it will apply non-maximum suppression to the final
predictions.
nms_iou_threshold: Intersection over union threshold used in non-maximum
suppression.
"""
if not is_training:
# Squeeze output voxel properties.
for key in standard_fields.get_output_voxel_fields():
if key in outputs and outputs[key] is not None:
outputs[key] = tf.squeeze(outputs[key], axis=0)
# Squeeze output point properties.
for key in standard_fields.get_output_point_fields():
if key in outputs and outputs[key] is not None:
outputs[key] = tf.squeeze(outputs[key], axis=0)
# Squeeze output object properties.
for key in standard_fields.get_output_object_fields():
if key in outputs and outputs[key] is not None:
outputs[key] = tf.squeeze(outputs[key], axis=0)
# Mask the valid voxels
mask_valid_voxels(inputs=inputs, outputs=outputs)
# Mask the valid points
mask_valid_points(inputs=inputs, outputs=outputs)
# NMS
postprocessor.postprocess(
outputs=outputs,
num_furthest_voxel_samples=num_furthest_voxel_samples,
sampler_score_vs_distance_coef=sampler_score_vs_distance_coef,
embedding_similarity_strategy=embedding_similarity_strategy,
embedding_similarity_threshold=embedding_similarity_threshold,
apply_nms=apply_nms,
nms_score_threshold=score_threshold,
nms_iou_threshold=nms_iou_threshold)
# Add instance segment point masks at eval time
if standard_fields.InputDataFields.points_to_voxel_mapping in inputs:
instance_segments_point_mask = (
voxel_utils.sparse_voxel_grid_to_pointcloud(
voxel_features=tf.expand_dims(tf.transpose(
outputs[standard_fields.DetectionResultFields.
instance_segments_voxel_mask]),
axis=0),
segment_ids=inputs[standard_fields.InputDataFields.
points_to_voxel_mapping],
num_valid_voxels=inputs[
standard_fields.InputDataFields.num_valid_voxels],
num_valid_points=inputs[
standard_fields.InputDataFields.num_valid_points]))
outputs[standard_fields.DetectionResultFields.
instance_segments_point_mask] = tf.transpose(
tf.squeeze(instance_segments_point_mask, axis=0))