def main(): parser = argparse.ArgumentParser( description="Evaluate one scene against ground truth") parser.add_argument("gt_scene", help="path to ground truth scene") parser.add_argument("sub_scene", help="path to submitted scene") args = parser.parse_args() (gt_dir, gt_scene_name) = os.path.split(args.gt_scene) (sub_dir, sub_scene_name) = os.path.split(args.sub_scene) gt_scene = ProjectScene.load(gt_dir, gt_scene_name) sub_scene = ProjectScene.load(sub_dir, sub_scene_name) if gt_scene.project_type != sub_scene.project_type: raise ValueError( 'Submission and ground truth project must be the same type.') evaluator_lut = { "meshes": MeshEvaluator, "voxels": VoxelEvaluator, "bounding_box": BBEvaluator, } eval_class = evaluator_lut[gt_scene.project_type] evaluator = eval_class(sub_scene, gt_scene, Evaluator.default_settings()) result = evaluator.evaluate_all() print("Shape Score (1 is best): {}\n\ Translation Error (0 is best): {}\n\ Rotation Error (0 is best): {}\n\ Semantics Score (1 is best): {}\n\ Perceptual Score (1 is best): {}\n".format(result["shape_score"], result["translation_error"], result["rotation_error"], result["semantics_score"], result["perceptual_score"]))
def setUp(self): """ Common setup for test cases """ self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data') self.ground_truth = ProjectScene.load(self.data_path, 'bounding_box_sample') self.submission = ProjectScene.load(self.data_path, 'bounding_box_sample') self.settings = Evaluator.default_settings() self.settings["categories"] = [ 'wall', 'floor', 'ceiling', 'sofa', 'coffee_table', 'beam']
def test_pose_error(self): """ Test rotation and translation error metric. """ self.ground_truth = ProjectScene.load(self.data_path, 'bounding_box_sample2') self.submission = ProjectScene.load(self.data_path, 'bounding_box_sample2') self.settings.thresholds = [0.5] # verify that correct pose is ok evaluator = BBEvaluator(self.submission, self.ground_truth, self.settings) rotation_error, translation_error = evaluator.pose_error() self.assertAlmostEqual(rotation_error, 0) self.assertAlmostEqual(translation_error, 0) # verify that rotation by symmetry amount is ok pose_orig = self.submission.elements["1069"].pose new_pose = Pose3(R=pose_orig.R * Rot3.Ry(math.pi), t=pose_orig.t) self.submission.elements["1069"].pose = new_pose evaluator = BBEvaluator(self.submission, self.ground_truth, self.settings) rotation_error, translation_error = evaluator.pose_error() self.assertAlmostEqual(rotation_error, 0) self.assertAlmostEqual(translation_error, 0) # verify that rotation by non-symmetry amount give correct error new_pose = Pose3(R=pose_orig.R * Rot3.Ry(math.radians(10)), t=pose_orig.t) self.submission.elements["1069"].pose = new_pose evaluator = BBEvaluator(self.submission, self.ground_truth, self.settings) rotation_error, translation_error = evaluator.pose_error() self.assertAlmostEqual(rotation_error, math.radians(10)) self.assertAlmostEqual(translation_error, 0) # verify that translation gives translation error new_pose = Pose3(R=pose_orig.R, t=pose_orig.t + [0.05, 0, 0]) self.submission.elements["1069"].pose = new_pose evaluator = BBEvaluator(self.submission, self.ground_truth, self.settings) rotation_error, translation_error = evaluator.pose_error() self.assertAlmostEqual(rotation_error, 0) self.assertAlmostEqual(translation_error, 0.05) # verify that empty sumission gives None, None new_pose = Pose3(R=pose_orig.R, t=pose_orig.t + [1, 0, 0]) self.submission.elements["1069"].pose = new_pose evaluator = BBEvaluator(self.submission, self.ground_truth, self.settings) rotation_error, translation_error = evaluator.pose_error() self.assertEqual(rotation_error, None) self.assertEqual(translation_error, None)
def setUp(self): """ Common setup for test cases """ self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data') self.ground_truth = ProjectScene.load(self.data_path, 'voxels_sample') self.submission = ProjectScene.load(self.data_path, 'voxels_sample') self.settings = Evaluator.default_settings() self.settings["categories"] = [ 'wall', 'floor', 'ceiling', 'sofa', 'coffee_table' ] self.settings["density"] = 100 self._started_at = time.time()
def test_visualize_mesh(self): visualize = False self.data_path = os.path.join(os.getcwd(), 'sumo/metrics/test_data') self.ground_truth = ProjectScene.load(self.data_path, 'meshes_sample') project_object = next(iter(self.ground_truth.elements.values())) mesh = next(iter(project_object.meshes.primitive_meshes())) utils.visualize_mesh(mesh, visualize)
def test_shape_score(self): """ Test class-specific shape score metric """ # verify no offset gives sim = 1 evaluator = BBEvaluator(self.submission, self.ground_truth, self.settings) shape_similarity = evaluator.shape_score() self.assertAlmostEqual(shape_similarity, 1) # verify that no submission gives sim = 0 scene = ProjectScene("bounding_box") evaluator2 = BBEvaluator(scene, self.ground_truth, self.settings) semantic_score = evaluator2.shape_score() self.assertEqual(semantic_score, 0) # verify that missed detection gives sim < 1 self.submission.elements.pop("1069") evaluator3 = BBEvaluator(self.submission, self.ground_truth, self.settings) semantic_score = evaluator3.shape_score() self.assertTrue(semantic_score < 1) # verify that extra detection gives sim < 1 self.ground_truth.elements.pop("57") self.ground_truth.elements.pop("1069") evaluator4 = BBEvaluator(self.submission, self.ground_truth, self.settings) semantic_score = evaluator4.shape_score() self.assertTrue(semantic_score < 1)
def test_shape_similarity(self): """ Verify that the shape similarity measure is producing sane outputs. """ # make a dummy scene scene = ProjectScene("bounding_box") evaluator = BBEvaluator(scene, scene, self.settings) obj1 = next(iter(self.submission.elements.values())) # ::: Temp only, use copy of obj1 if deep copy can be made to work obj2 = next(iter(self.ground_truth.elements.values())) # verify no offset gives sim = 1 sim = evaluator._shape_similarity(obj1, obj2) self.assertAlmostEqual(sim, 1) # verify small offset gives sim between 0 and 1 pose_orig = obj2.pose obj2.pose = Pose3(t=pose_orig.t + [0.1, 0, 0], R=pose_orig.R) sim = evaluator._shape_similarity(obj1, obj2) self.assertTrue(sim < 1 and sim > 0) # verify large offset gives sim = 0 obj2.pose = Pose3(t=pose_orig.t + [5, 5, 5], R=pose_orig.R) sim = evaluator._shape_similarity(obj1, obj2) self.assertAlmostEqual(sim, 0)
def test_shape_similarity(self): """ Verify that the shape similarity measure is producing sane outputs. """ # make a dummy scene scene = ProjectScene("voxels") # TODO: Get deepcopy working for ProjectScene and make a simpler example for faster unit test. evaluator = VoxelEvaluator(self.submission, self.ground_truth, self.settings) obj1 = self.submission.elements["1069"] obj2 = self.ground_truth.elements["1069"] # verify no offset gives sim = 1 sim = evaluator._shape_similarity(obj1, obj2) self.assertAlmostEqual(sim, 1) # verify small offset gives sim between 0 and 1 voxel_centers_orig = obj2.voxel_centers obj2.voxel_centers = obj2.voxel_centers + np.array([0.2, 0, 0]) sim = evaluator._shape_similarity(obj1, obj2) self.assertTrue(sim < 1 and sim > 0) # verify large offset gives sim = 0 obj2.voxel_centers = obj2.voxel_centers + np.array([1, 0, 0]) sim = evaluator._shape_similarity(obj1, obj2) self.assertAlmostEqual(sim, 0) obj2.voxel_centers = voxel_centers_orig shape_score = evaluator.shape_score() self.assertAlmostEqual(shape_score, 1)
def run(self, project, target_type): """ Convert an in-memory project to the target type Inputs: project (ProjectScene) - input project target_type (string) - voxels or bounding_box Return: new_project (ProjectScene) - a project with the target project type Exceptions: ValueError - if target_type is not allowed for the given input project. See above for allowed conversions. """ if (project.project_type, target_type) not in self.allowed_conversions: raise ValueError("Invalid target_type ({}) for \ project with type {}".format(target_type, project.project_type)) new_settings = deepcopy(project.settings) new_elements = ProjectObjectDict() for element in project.elements.values(): new_element = self.convert_element(element, target_type) new_elements[new_element.id] = new_element new_project = ProjectScene(project_type=target_type, elements=new_elements, settings=new_settings) return new_project
def test_bounding_box_io(self): """ Save and load a bounding_box project. Also try overwriting the project (should fail). """ project_scene = ProjectScene(project_type="bounding_box") bounds = Box3d([-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]) po = ProjectObject(id="1", bounds=bounds, category="chair") project_scene.elements["1"] = po # test saving project_scene.save(path=self.temp_directory, project_name="test") xml_path = os.path.join(self.temp_directory, "test", "test.xml") self.assertTrue(os.path.isfile(xml_path)) # test overwriting self.assertRaises(OSError, project_scene.save, path=self.temp_directory, project_name="test") # test loading project_scene = ProjectScene.load(path=self.temp_directory, project_name="test") self.assertIsInstance(project_scene, ProjectScene) self.assertIsInstance(project_scene.elements, ProjectObjectDict) # ::: TODO: improve check with equality test on project_scene # Check bounding box for the first ProjectObject po = project_scene.elements["1"] self.assertTrue(po.bounds.almost_equal(bounds, atol=0.01)) self.assertEqual(po.category, "chair")
def test_voxel_to_bbox(self): """ Conversion from voxel to bbox. Test number of elements and project_type. Does not test contents for accuracy. """ voxels_model = ProjectScene.load(TEST_PATH, "voxels_sample") bbox_model = ProjectConverter().run(voxels_model, "bounding_box") self.assertEqual(bbox_model.project_type, "bounding_box") self.assertEqual(len(bbox_model.elements), len(voxels_model.elements)) for element in bbox_model.elements.values(): self.assertTrue(hasattr(element, "bounds"))
def test_meshes_to_voxels(self): """ Conversion from meshes to voxels. Test number of elements and project_type. Does not test contents for accuracy. """ meshes_model = ProjectScene.load(TEST_PATH, "meshes_sample") voxel_model = ProjectConverter().run(meshes_model, "voxels") self.assertEqual(voxel_model.project_type, "voxels") self.assertEqual(len(voxel_model.elements), len(meshes_model.elements)) for element in voxel_model.elements.values(): self.assertTrue(hasattr(element, "voxels"))
def test_invalid_conversions(self): """ Make sure invalid conversions raise an error """ bbox_model = ProjectScene.load(TEST_PATH, "bounding_box_sample") self.assertRaises(ValueError, ProjectConverter().run, project=bbox_model, target_type="voxels") self.assertRaises(ValueError, ProjectConverter().run, project=bbox_model, target_type="meshes")
from sumo.geometry.rot3 import Rot3, ENU_R_CAMERA import numpy as np wRc = np.transpose(np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]], dtype=float)) rot = Rot3(wRc) print(rot.matrix()) # MultiImageTiff from sumo.geometry.inverse_depth import depth_image_of_inverse_depth_map from sumo.images.multi_image_tiff import MultiImageTiff, MultiImagePageType tiff_path = parutil.get_file_path( '/mnt/lustre/sunjiankai/Dataset/sample_data/sumo-input/sumo-input.tif') multi = MultiImageTiff.load(tiff_path) print('multi.rgb.shape: ', multi.rgb.shape, 'multi.range.shape (Depth): ', multi.range.shape, 'multi.category.shape (Category): ', multi.category.shape, 'multi.instance.shape (Instance): ', multi.instance.shape) # multi.rgb.shape: (1024, 6144, 3) multi.range.shape: (1024, 6144) multi.category.shape: (1024, 6144) multi.instance.shape: (1024, 6144) from sumo.semantic.project_converter import ProjectConverter from sumo.semantic.project_scene import ProjectScene glb_path = parutil.get_file_path( '/mnt/lustre/sunjiankai/Dataset/sample_data/sumo-output') meshes_model = ProjectScene.load(glb_path, "bounding_box_sample") bbox_model = ProjectConverter().run(meshes_model, "bounding_box") print('bbox_model.elements[\'1087\'].bounds.corners():\n', bbox_model.elements['1087'].bounds.corners()) print('bbox_model.elements[\'1087\'].pose.t', bbox_model.elements['1087'].pose.t)