def test_sparse_voxel_grid_to_pointcloud(self): voxel_features_0 = tf.constant( [[0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 1.0]], dtype=tf.float32) voxel_features_1 = tf.constant( [[0.0, 0.0, 0.5], [0.0, 0.5, 0.0], [0.5, 0.0, 0.0], [0.0, 0.0, 0.0], [0.5, 0.5, 0.5]], dtype=tf.float32) voxel_features = tf.stack([voxel_features_0, voxel_features_1], axis=0) segment_ids = tf.constant( [[0, 0, 1, 1, 2, 2, 0, 0, 0, 0], [1, 3, 1, 2, 0, 4, 4, 0, 0, 0]], dtype=tf.int32) num_valid_voxels = tf.constant([3, 5], dtype=tf.int32) num_valid_points = tf.constant([7, 9], dtype=tf.int32) point_features = voxel_utils.sparse_voxel_grid_to_pointcloud( voxel_features=voxel_features, segment_ids=segment_ids, num_valid_voxels=num_valid_voxels, num_valid_points=num_valid_points) np_point_features = point_features.numpy() self.assertAllEqual(np_point_features.shape, [2, 10, 3]) self.assertAllClose( np_point_features[0], np.array([[0.0, 0.0, 1.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])) self.assertAllClose( np_point_features[1], np.array([[0.0, 0.5, 0.0], [0.0, 0.0, 0.0], [0.0, 0.5, 0.0], [0.5, 0.0, 0.0], [0.0, 0.0, 0.5], [0.5, 0.5, 0.5], [0.5, 0.5, 0.5], [0.0, 0.0, 0.5], [0.0, 0.0, 0.5], [0.0, 0.0, 0.0]]))
def __call__(self, inputs, training=True): """Runs the model and returns the semantic logits prediction. Args: inputs: A dictionary of tensors containing inputs for the model. training: Whether the model runs in training mode. Returns: A dictionary of tensors containing semantic logits prediction. """ inputs[standard_fields.InputDataFields.num_valid_voxels] = tf.reshape( inputs[standard_fields.InputDataFields.num_valid_voxels], [-1]) voxel_inputs = ( inputs[standard_fields.InputDataFields.voxel_features], inputs[standard_fields.InputDataFields.voxel_xyz_indices], inputs[standard_fields.InputDataFields.num_valid_voxels]) outputs = self.sparse_conv_unet(voxel_inputs, training=training) # If at eval time, transfer voxel features to points if ((not training) and (standard_fields.InputDataFields.points_to_voxel_mapping in inputs)): voxel_to_point_mapping = ( standard_fields.get_output_voxel_to_point_field_mapping()) point_tensor_outputs = {} for task_name in outputs: if task_name in voxel_to_point_mapping and outputs[ task_name] is not None: point_tensor_outputs[voxel_to_point_mapping[task_name]] = ( voxel_utils.sparse_voxel_grid_to_pointcloud( voxel_features=outputs[task_name], segment_ids=inputs[standard_fields.InputDataFields. points_to_voxel_mapping], num_valid_voxels=inputs[ standard_fields.InputDataFields. num_valid_voxels], num_valid_points=inputs[ standard_fields.InputDataFields. num_valid_points])) outputs.update(point_tensor_outputs) model_utils.postprocess( inputs=inputs, outputs=outputs, is_training=training, num_furthest_voxel_samples=self.num_furthest_voxel_samples, sampler_score_vs_distance_coef=self.sampler_score_vs_distance_coef, embedding_similarity_strategy=self.embedding_similarity_strategy, embedding_similarity_threshold=self.embedding_similarity_threshold, score_threshold=self.nms_score_threshold, apply_nms=self.apply_nms, nms_iou_threshold=self.nms_iou_threshold) if training: self.calculate_losses(inputs=inputs, outputs=outputs) return outputs
def call(self, inputs, training=True): """Runs the model and returns the semantic logits prediction. Args: inputs: A dictionary of tensors containing inputs for the model. training: Whether the model runs in training mode. Returns: A dictionary of tensors containing semantic logits prediction. """ # when not using custom train step, this field becomes 2 dim. inputs[standard_fields.InputDataFields.num_valid_voxels] = tf.reshape( inputs[standard_fields.InputDataFields.num_valid_voxels], [-1]) voxel_inputs = ( inputs[standard_fields.InputDataFields.voxel_features], inputs[standard_fields.InputDataFields.voxel_xyz_indices], inputs[standard_fields.InputDataFields.num_valid_voxels]) outputs = self.sparse_conv_unet(voxel_inputs, training=training) # If at eval time, transfer voxel features to points if ((not training) and (standard_fields.InputDataFields.points_to_voxel_mapping in inputs)): inputs[ standard_fields.InputDataFields.num_valid_points] = tf.reshape( inputs[standard_fields.InputDataFields.num_valid_points], [-1]) voxel_to_point_mapping = ( standard_fields.get_output_voxel_to_point_field_mapping()) point_tensor_outputs = {} for task_name in outputs: if task_name in voxel_to_point_mapping and outputs[ task_name] is not None: point_tensor_outputs[voxel_to_point_mapping[task_name]] = ( voxel_utils.sparse_voxel_grid_to_pointcloud( voxel_features=outputs[task_name], segment_ids=inputs[standard_fields.InputDataFields. points_to_voxel_mapping], num_valid_voxels=inputs[ standard_fields.InputDataFields. num_valid_voxels], num_valid_points=inputs[ standard_fields.InputDataFields. num_valid_points])) # include fields used by tensorboard visualization call back. outputs.update(point_tensor_outputs) if training: self.calculate_losses(inputs=inputs, outputs=outputs) return outputs
def __call__(self, inputs, training=True): """Runs the model and returns the semantic logits prediciton. Args: inputs: A dictionary of tensors containing inputs for the model. training: Whether the model runs in training mode. Returns: A dictionary of tensors containing semantic logits prediciton. """ # when not using custom train step, this field becomes 2 dim. inputs[standard_fields.InputDataFields.num_valid_voxels] = tf.reshape( inputs[standard_fields.InputDataFields.num_valid_voxels], [-1]) voxel_inputs = ( inputs[standard_fields.InputDataFields.voxel_features], inputs[standard_fields.InputDataFields.voxel_xyz_indices], inputs[standard_fields.InputDataFields.num_valid_voxels]) outputs = self.sparse_conv_hourglass(voxel_inputs, training=training) outputs[standard_fields.DetectionResultFields. object_center_voxels] += inputs[ standard_fields.InputDataFields.voxel_positions] model_utils.rectify_outputs(outputs=outputs) # If at eval time, transfer voxel features to points if ((not training) and (standard_fields.InputDataFields.points_to_voxel_mapping in inputs)): inputs[ standard_fields.InputDataFields.num_valid_points] = tf.reshape( inputs[standard_fields.InputDataFields.num_valid_points], [-1]) voxel_to_point_mapping = ( standard_fields.get_output_voxel_to_point_field_mapping()) point_tensor_outputs = {} for task_name in outputs: if task_name in voxel_to_point_mapping and outputs[ task_name] is not None: point_tensor_outputs[voxel_to_point_mapping[task_name]] = ( voxel_utils.sparse_voxel_grid_to_pointcloud( voxel_features=outputs[task_name], segment_ids=inputs[standard_fields.InputDataFields. points_to_voxel_mapping], num_valid_voxels=inputs[ standard_fields.InputDataFields. num_valid_voxels], num_valid_points=inputs[ standard_fields.InputDataFields. num_valid_points])) outputs.update(point_tensor_outputs) model_utils.postprocess( inputs=inputs, outputs=outputs, is_training=training, apply_nms=self.apply_nms, nms_score_threshold=self.nms_score_threshold, nms_iou_threshold=self.nms_iou_threshold, nms_max_num_predicted_boxes=self.nms_max_num_predicted_boxes, use_furthest_voxel_sampling=self.use_furthest_voxel_sampling, num_furthest_voxel_samples=self.num_furthest_voxel_samples, sampler_score_vs_distance_coef=self.sampler_score_vs_distance_coef) if training: self.calculate_losses(inputs=inputs, outputs=outputs) return outputs
def postprocess(inputs, outputs, is_training, num_furthest_voxel_samples, sampler_score_vs_distance_coef, embedding_similarity_strategy, embedding_similarity_threshold, score_threshold, apply_nms, nms_iou_threshold): """Post-processor function. Args: inputs: A dictionary containing input tensors. outputs: A dictionary containing predicted tensors. is_training: If during training stage or not. num_furthest_voxel_samples: Number of voxels to be sampled using furthest voxel sampling in the postprocessor. sampler_score_vs_distance_coef: The coefficient that balances the weight between furthest voxel sampling and highest score sampling in the postprocessor. embedding_similarity_strategy: Embedding similarity strategy. embedding_similarity_threshold: Similarity threshold used to decide if two point embedding vectors belong to the same instance. score_threshold: Instance score threshold used throughout postprocessing. apply_nms: If True, it will apply non-maximum suppression to the final predictions. nms_iou_threshold: Intersection over union threshold used in non-maximum suppression. """ if not is_training: # Squeeze output voxel properties. for key in standard_fields.get_output_voxel_fields(): if key in outputs and outputs[key] is not None: outputs[key] = tf.squeeze(outputs[key], axis=0) # Squeeze output point properties. for key in standard_fields.get_output_point_fields(): if key in outputs and outputs[key] is not None: outputs[key] = tf.squeeze(outputs[key], axis=0) # Squeeze output object properties. for key in standard_fields.get_output_object_fields(): if key in outputs and outputs[key] is not None: outputs[key] = tf.squeeze(outputs[key], axis=0) # Mask the valid voxels mask_valid_voxels(inputs=inputs, outputs=outputs) # Mask the valid points mask_valid_points(inputs=inputs, outputs=outputs) # NMS postprocessor.postprocess( outputs=outputs, num_furthest_voxel_samples=num_furthest_voxel_samples, sampler_score_vs_distance_coef=sampler_score_vs_distance_coef, embedding_similarity_strategy=embedding_similarity_strategy, embedding_similarity_threshold=embedding_similarity_threshold, apply_nms=apply_nms, nms_score_threshold=score_threshold, nms_iou_threshold=nms_iou_threshold) # Add instance segment point masks at eval time if standard_fields.InputDataFields.points_to_voxel_mapping in inputs: instance_segments_point_mask = ( voxel_utils.sparse_voxel_grid_to_pointcloud( voxel_features=tf.expand_dims(tf.transpose( outputs[standard_fields.DetectionResultFields. instance_segments_voxel_mask]), axis=0), segment_ids=inputs[standard_fields.InputDataFields. points_to_voxel_mapping], num_valid_voxels=inputs[ standard_fields.InputDataFields.num_valid_voxels], num_valid_points=inputs[ standard_fields.InputDataFields.num_valid_points])) outputs[standard_fields.DetectionResultFields. instance_segments_point_mask] = tf.transpose( tf.squeeze(instance_segments_point_mask, axis=0))