def main():
parser = argparse.ArgumentParser(
description="Evaluate one scene against ground truth")
parser.add_argument("gt_scene", help="path to ground truth scene")
parser.add_argument("sub_scene", help="path to submitted scene")
args = parser.parse_args()
(gt_dir, gt_scene_name) = os.path.split(args.gt_scene)
(sub_dir, sub_scene_name) = os.path.split(args.sub_scene)
gt_scene = ProjectScene.load(gt_dir, gt_scene_name)
sub_scene = ProjectScene.load(sub_dir, sub_scene_name)
if gt_scene.project_type != sub_scene.project_type:
raise ValueError(
'Submission and ground truth project must be the same type.')
evaluator_lut = {
"meshes": MeshEvaluator,
"voxels": VoxelEvaluator,
"bounding_box": BBEvaluator,
}
eval_class = evaluator_lut[gt_scene.project_type]
evaluator = eval_class(sub_scene, gt_scene, Evaluator.default_settings())
result = evaluator.evaluate_all()
print("Shape Score (1 is best): {}\n\
Translation Error (0 is best): {}\n\
Rotation Error (0 is best): {}\n\
Semantics Score (1 is best): {}\n\
Perceptual Score (1 is best): {}\n".format(result["shape_score"],
result["translation_error"],
result["rotation_error"],
result["semantics_score"],
result["perceptual_score"]))
def setUp(self):
"""
Common setup for test cases
"""
self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data')
self.ground_truth = ProjectScene.load(self.data_path, 'bounding_box_sample')
self.submission = ProjectScene.load(self.data_path, 'bounding_box_sample')
self.settings = Evaluator.default_settings()
self.settings["categories"] = [
'wall', 'floor', 'ceiling', 'sofa', 'coffee_table', 'beam']
def test_pose_error(self):
"""
Test rotation and translation error metric.
"""
self.ground_truth = ProjectScene.load(self.data_path,
'bounding_box_sample2')
self.submission = ProjectScene.load(self.data_path,
'bounding_box_sample2')
self.settings.thresholds = [0.5]
# verify that correct pose is ok
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, 0)
self.assertAlmostEqual(translation_error, 0)
# verify that rotation by symmetry amount is ok
pose_orig = self.submission.elements["1069"].pose
new_pose = Pose3(R=pose_orig.R * Rot3.Ry(math.pi), t=pose_orig.t)
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, 0)
self.assertAlmostEqual(translation_error, 0)
# verify that rotation by non-symmetry amount give correct error
new_pose = Pose3(R=pose_orig.R * Rot3.Ry(math.radians(10)),
t=pose_orig.t)
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, math.radians(10))
self.assertAlmostEqual(translation_error, 0)
# verify that translation gives translation error
new_pose = Pose3(R=pose_orig.R, t=pose_orig.t + [0.05, 0, 0])
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, 0)
self.assertAlmostEqual(translation_error, 0.05)
# verify that empty sumission gives None, None
new_pose = Pose3(R=pose_orig.R, t=pose_orig.t + [1, 0, 0])
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertEqual(rotation_error, None)
self.assertEqual(translation_error, None)
def setUp(self):
"""
Common setup for test cases
"""
self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data')
self.ground_truth = ProjectScene.load(self.data_path, 'voxels_sample')
self.submission = ProjectScene.load(self.data_path, 'voxels_sample')
self.settings = Evaluator.default_settings()
self.settings["categories"] = [
'wall', 'floor', 'ceiling', 'sofa', 'coffee_table'
]
self.settings["density"] = 100
self._started_at = time.time()
def test_visualize_mesh(self):
visualize = False
self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data')
self.ground_truth = ProjectScene.load(self.data_path, 'meshes_sample')
project_object = next(iter(self.ground_truth.elements.values()))
mesh = next(iter(project_object.meshes.primitive_meshes()))
utils.visualize_mesh(mesh, visualize)
def test_bounding_box_io(self):
"""
Save and load a bounding_box project. Also try overwriting the
project (should fail).
"""
project_scene = ProjectScene(project_type="bounding_box")
bounds = Box3d([-0.5, -0.5, -0.5], [0.5, 0.5, 0.5])
po = ProjectObject(id="1", bounds=bounds, category="chair")
project_scene.elements["1"] = po
# test saving
project_scene.save(path=self.temp_directory, project_name="test")
xml_path = os.path.join(self.temp_directory, "test", "test.xml")
self.assertTrue(os.path.isfile(xml_path))
# test overwriting
self.assertRaises(OSError,
project_scene.save,
path=self.temp_directory,
project_name="test")
# test loading
project_scene = ProjectScene.load(path=self.temp_directory,
project_name="test")
self.assertIsInstance(project_scene, ProjectScene)
self.assertIsInstance(project_scene.elements, ProjectObjectDict)
# ::: TODO: improve check with equality test on project_scene
# Check bounding box for the first ProjectObject
po = project_scene.elements["1"]
self.assertTrue(po.bounds.almost_equal(bounds, atol=0.01))
self.assertEqual(po.category, "chair")
def test_voxel_to_bbox(self):
"""
Conversion from voxel to bbox. Test number of elements
and project_type. Does not test contents for accuracy.
"""
voxels_model = ProjectScene.load(TEST_PATH, "voxels_sample")
bbox_model = ProjectConverter().run(voxels_model, "bounding_box")
self.assertEqual(bbox_model.project_type, "bounding_box")
self.assertEqual(len(bbox_model.elements), len(voxels_model.elements))
for element in bbox_model.elements.values():
self.assertTrue(hasattr(element, "bounds"))
def test_meshes_to_voxels(self):
"""
Conversion from meshes to voxels. Test number of elements
and project_type. Does not test contents for accuracy.
"""
meshes_model = ProjectScene.load(TEST_PATH, "meshes_sample")
voxel_model = ProjectConverter().run(meshes_model, "voxels")
self.assertEqual(voxel_model.project_type, "voxels")
self.assertEqual(len(voxel_model.elements), len(meshes_model.elements))
for element in voxel_model.elements.values():
self.assertTrue(hasattr(element, "voxels"))
def test_invalid_conversions(self):
"""
Make sure invalid conversions raise an error
"""
bbox_model = ProjectScene.load(TEST_PATH, "bounding_box_sample")
self.assertRaises(ValueError,
ProjectConverter().run,
project=bbox_model,
target_type="voxels")
self.assertRaises(ValueError,
ProjectConverter().run,
project=bbox_model,
target_type="meshes")
from sumo.geometry.rot3 import Rot3, ENU_R_CAMERA
import numpy as np
wRc = np.transpose(np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]], dtype=float))
rot = Rot3(wRc)
print(rot.matrix())
# MultiImageTiff
from sumo.geometry.inverse_depth import depth_image_of_inverse_depth_map
from sumo.images.multi_image_tiff import MultiImageTiff, MultiImagePageType
tiff_path = parutil.get_file_path(
'/mnt/lustre/sunjiankai/Dataset/sample_data/sumo-input/sumo-input.tif')
multi = MultiImageTiff.load(tiff_path)
print('multi.rgb.shape: ', multi.rgb.shape, 'multi.range.shape (Depth): ',
multi.range.shape, 'multi.category.shape (Category): ',
multi.category.shape, 'multi.instance.shape (Instance): ',
multi.instance.shape)
# multi.rgb.shape: (1024, 6144, 3) multi.range.shape: (1024, 6144) multi.category.shape: (1024, 6144) multi.instance.shape: (1024, 6144)
from sumo.semantic.project_converter import ProjectConverter
from sumo.semantic.project_scene import ProjectScene
glb_path = parutil.get_file_path(
'/mnt/lustre/sunjiankai/Dataset/sample_data/sumo-output')
meshes_model = ProjectScene.load(glb_path, "bounding_box_sample")
bbox_model = ProjectConverter().run(meshes_model, "bounding_box")
print('bbox_model.elements[\'1087\'].bounds.corners():\n',
bbox_model.elements['1087'].bounds.corners())
print('bbox_model.elements[\'1087\'].pose.t',
bbox_model.elements['1087'].pose.t)
