def main():
parser = argparse.ArgumentParser(
description="Evaluate one scene against ground truth")
parser.add_argument("gt_scene", help="path to ground truth scene")
parser.add_argument("sub_scene", help="path to submitted scene")
args = parser.parse_args()
(gt_dir, gt_scene_name) = os.path.split(args.gt_scene)
(sub_dir, sub_scene_name) = os.path.split(args.sub_scene)
gt_scene = ProjectScene.load(gt_dir, gt_scene_name)
sub_scene = ProjectScene.load(sub_dir, sub_scene_name)
if gt_scene.project_type != sub_scene.project_type:
raise ValueError(
'Submission and ground truth project must be the same type.')
evaluator_lut = {
"meshes": MeshEvaluator,
"voxels": VoxelEvaluator,
"bounding_box": BBEvaluator,
}
eval_class = evaluator_lut[gt_scene.project_type]
evaluator = eval_class(sub_scene, gt_scene, Evaluator.default_settings())
result = evaluator.evaluate_all()
print("Shape Score (1 is best): {}\n\
Translation Error (0 is best): {}\n\
Rotation Error (0 is best): {}\n\
Semantics Score (1 is best): {}\n\
Perceptual Score (1 is best): {}\n".format(result["shape_score"],
result["translation_error"],
result["rotation_error"],
result["semantics_score"],
result["perceptual_score"]))
def setUp(self):
"""
Common setup for test cases
"""
self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data')
self.ground_truth = ProjectScene.load(self.data_path, 'bounding_box_sample')
self.submission = ProjectScene.load(self.data_path, 'bounding_box_sample')
self.settings = Evaluator.default_settings()
self.settings["categories"] = [
'wall', 'floor', 'ceiling', 'sofa', 'coffee_table', 'beam']
def test_pose_error(self):
"""
Test rotation and translation error metric.
"""
self.ground_truth = ProjectScene.load(self.data_path,
'bounding_box_sample2')
self.submission = ProjectScene.load(self.data_path,
'bounding_box_sample2')
self.settings.thresholds = [0.5]
# verify that correct pose is ok
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, 0)
self.assertAlmostEqual(translation_error, 0)
# verify that rotation by symmetry amount is ok
pose_orig = self.submission.elements["1069"].pose
new_pose = Pose3(R=pose_orig.R * Rot3.Ry(math.pi), t=pose_orig.t)
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, 0)
self.assertAlmostEqual(translation_error, 0)
# verify that rotation by non-symmetry amount give correct error
new_pose = Pose3(R=pose_orig.R * Rot3.Ry(math.radians(10)),
t=pose_orig.t)
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, math.radians(10))
self.assertAlmostEqual(translation_error, 0)
# verify that translation gives translation error
new_pose = Pose3(R=pose_orig.R, t=pose_orig.t + [0.05, 0, 0])
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertAlmostEqual(rotation_error, 0)
self.assertAlmostEqual(translation_error, 0.05)
# verify that empty sumission gives None, None
new_pose = Pose3(R=pose_orig.R, t=pose_orig.t + [1, 0, 0])
self.submission.elements["1069"].pose = new_pose
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
rotation_error, translation_error = evaluator.pose_error()
self.assertEqual(rotation_error, None)
self.assertEqual(translation_error, None)
def setUp(self):
"""
Common setup for test cases
"""
self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data')
self.ground_truth = ProjectScene.load(self.data_path, 'voxels_sample')
self.submission = ProjectScene.load(self.data_path, 'voxels_sample')
self.settings = Evaluator.default_settings()
self.settings["categories"] = [
'wall', 'floor', 'ceiling', 'sofa', 'coffee_table'
]
self.settings["density"] = 100
self._started_at = time.time()
def test_visualize_mesh(self):
visualize = False
self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data')
self.ground_truth = ProjectScene.load(self.data_path, 'meshes_sample')
project_object = next(iter(self.ground_truth.elements.values()))
mesh = next(iter(project_object.meshes.primitive_meshes()))
utils.visualize_mesh(mesh, visualize)
def test_shape_score(self):
"""
Test class-specific shape score metric
"""
# verify no offset gives sim = 1
evaluator = BBEvaluator(self.submission, self.ground_truth,
self.settings)
shape_similarity = evaluator.shape_score()
self.assertAlmostEqual(shape_similarity, 1)
# verify that no submission gives sim = 0
scene = ProjectScene("bounding_box")
evaluator2 = BBEvaluator(scene, self.ground_truth, self.settings)
semantic_score = evaluator2.shape_score()
self.assertEqual(semantic_score, 0)
# verify that missed detection gives sim < 1
self.submission.elements.pop("1069")
evaluator3 = BBEvaluator(self.submission, self.ground_truth,
self.settings)
semantic_score = evaluator3.shape_score()
self.assertTrue(semantic_score < 1)
# verify that extra detection gives sim < 1
self.ground_truth.elements.pop("57")
self.ground_truth.elements.pop("1069")
evaluator4 = BBEvaluator(self.submission, self.ground_truth,
self.settings)
semantic_score = evaluator4.shape_score()
self.assertTrue(semantic_score < 1)
def test_shape_similarity(self):
"""
Verify that the shape similarity measure is producing sane outputs.
"""
# make a dummy scene
scene = ProjectScene("bounding_box")
evaluator = BBEvaluator(scene, scene, self.settings)
obj1 = next(iter(self.submission.elements.values()))
# ::: Temp only, use copy of obj1 if deep copy can be made to work
obj2 = next(iter(self.ground_truth.elements.values()))
# verify no offset gives sim = 1
sim = evaluator._shape_similarity(obj1, obj2)
self.assertAlmostEqual(sim, 1)
# verify small offset gives sim between 0 and 1
pose_orig = obj2.pose
obj2.pose = Pose3(t=pose_orig.t + [0.1, 0, 0], R=pose_orig.R)
sim = evaluator._shape_similarity(obj1, obj2)
self.assertTrue(sim < 1 and sim > 0)
# verify large offset gives sim = 0
obj2.pose = Pose3(t=pose_orig.t + [5, 5, 5], R=pose_orig.R)
sim = evaluator._shape_similarity(obj1, obj2)
self.assertAlmostEqual(sim, 0)
def test_shape_similarity(self):
"""
Verify that the shape similarity measure is producing sane outputs.
"""
# make a dummy scene
scene = ProjectScene("voxels")
# TODO: Get deepcopy working for ProjectScene and make a simpler example for faster unit test.
evaluator = VoxelEvaluator(self.submission, self.ground_truth,
self.settings)
obj1 = self.submission.elements["1069"]
obj2 = self.ground_truth.elements["1069"]
# verify no offset gives sim = 1
sim = evaluator._shape_similarity(obj1, obj2)
self.assertAlmostEqual(sim, 1)
# verify small offset gives sim between 0 and 1
voxel_centers_orig = obj2.voxel_centers
obj2.voxel_centers = obj2.voxel_centers + np.array([0.2, 0, 0])
sim = evaluator._shape_similarity(obj1, obj2)
self.assertTrue(sim < 1 and sim > 0)
# verify large offset gives sim = 0
obj2.voxel_centers = obj2.voxel_centers + np.array([1, 0, 0])
sim = evaluator._shape_similarity(obj1, obj2)
self.assertAlmostEqual(sim, 0)
obj2.voxel_centers = voxel_centers_orig
shape_score = evaluator.shape_score()
self.assertAlmostEqual(shape_score, 1)
def run(self, project, target_type):
"""
Convert an in-memory project to the target type
Inputs:
project (ProjectScene) - input project
target_type (string) - voxels or bounding_box
Return:
new_project (ProjectScene) - a project with the target project type
Exceptions:
ValueError - if target_type is not allowed for the given input project.
See above for allowed conversions.
"""
if (project.project_type, target_type) not in self.allowed_conversions:
raise ValueError("Invalid target_type ({}) for \
project with type {}".format(target_type,
project.project_type))
new_settings = deepcopy(project.settings)
new_elements = ProjectObjectDict()
for element in project.elements.values():
new_element = self.convert_element(element, target_type)
new_elements[new_element.id] = new_element
new_project = ProjectScene(project_type=target_type,
elements=new_elements,
settings=new_settings)
return new_project
def test_bounding_box_io(self):
"""
Save and load a bounding_box project. Also try overwriting the
project (should fail).
"""
project_scene = ProjectScene(project_type="bounding_box")
bounds = Box3d([-0.5, -0.5, -0.5], [0.5, 0.5, 0.5])
po = ProjectObject(id="1", bounds=bounds, category="chair")
project_scene.elements["1"] = po
# test saving
project_scene.save(path=self.temp_directory, project_name="test")
xml_path = os.path.join(self.temp_directory, "test", "test.xml")
self.assertTrue(os.path.isfile(xml_path))
# test overwriting
self.assertRaises(OSError,
project_scene.save,
path=self.temp_directory,
project_name="test")
# test loading
project_scene = ProjectScene.load(path=self.temp_directory,
project_name="test")
self.assertIsInstance(project_scene, ProjectScene)
self.assertIsInstance(project_scene.elements, ProjectObjectDict)
# ::: TODO: improve check with equality test on project_scene
# Check bounding box for the first ProjectObject
po = project_scene.elements["1"]
self.assertTrue(po.bounds.almost_equal(bounds, atol=0.01))
self.assertEqual(po.category, "chair")
def test_voxel_to_bbox(self):
"""
Conversion from voxel to bbox. Test number of elements
and project_type. Does not test contents for accuracy.
"""
voxels_model = ProjectScene.load(TEST_PATH, "voxels_sample")
bbox_model = ProjectConverter().run(voxels_model, "bounding_box")
self.assertEqual(bbox_model.project_type, "bounding_box")
self.assertEqual(len(bbox_model.elements), len(voxels_model.elements))
for element in bbox_model.elements.values():
self.assertTrue(hasattr(element, "bounds"))
def test_meshes_to_voxels(self):
"""
Conversion from meshes to voxels. Test number of elements
and project_type. Does not test contents for accuracy.
"""
meshes_model = ProjectScene.load(TEST_PATH, "meshes_sample")
voxel_model = ProjectConverter().run(meshes_model, "voxels")
self.assertEqual(voxel_model.project_type, "voxels")
self.assertEqual(len(voxel_model.elements), len(meshes_model.elements))
for element in voxel_model.elements.values():
self.assertTrue(hasattr(element, "voxels"))
def test_invalid_conversions(self):
"""
Make sure invalid conversions raise an error
"""
bbox_model = ProjectScene.load(TEST_PATH, "bounding_box_sample")
self.assertRaises(ValueError,
ProjectConverter().run,
project=bbox_model,
target_type="voxels")
self.assertRaises(ValueError,
ProjectConverter().run,
project=bbox_model,
target_type="meshes")
from sumo.geometry.rot3 import Rot3, ENU_R_CAMERA
import numpy as np
wRc = np.transpose(np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]], dtype=float))
rot = Rot3(wRc)
print(rot.matrix())
# MultiImageTiff
from sumo.geometry.inverse_depth import depth_image_of_inverse_depth_map
from sumo.images.multi_image_tiff import MultiImageTiff, MultiImagePageType
tiff_path = parutil.get_file_path(
'/mnt/lustre/sunjiankai/Dataset/sample_data/sumo-input/sumo-input.tif')
multi = MultiImageTiff.load(tiff_path)
print('multi.rgb.shape: ', multi.rgb.shape, 'multi.range.shape (Depth): ',
multi.range.shape, 'multi.category.shape (Category): ',
multi.category.shape, 'multi.instance.shape (Instance): ',
multi.instance.shape)
# multi.rgb.shape: (1024, 6144, 3) multi.range.shape: (1024, 6144) multi.category.shape: (1024, 6144) multi.instance.shape: (1024, 6144)
from sumo.semantic.project_converter import ProjectConverter
from sumo.semantic.project_scene import ProjectScene
glb_path = parutil.get_file_path(
'/mnt/lustre/sunjiankai/Dataset/sample_data/sumo-output')
meshes_model = ProjectScene.load(glb_path, "bounding_box_sample")
bbox_model = ProjectConverter().run(meshes_model, "bounding_box")
print('bbox_model.elements[\'1087\'].bounds.corners():\n',
bbox_model.elements['1087'].bounds.corners())
print('bbox_model.elements[\'1087\'].pose.t',
bbox_model.elements['1087'].pose.t)