def test_evaluate_all(self):
"""
Test the evaluate_all function by running it and checking the
resulting metric values. Only a subset of metrics are checked.
"""
evaluator = VoxelEvaluator(self.submission, self.ground_truth, self.settings)
metrics = evaluator.evaluate_all()
self.assertTrue(metrics["rms_points_error"] < 0.05)
self.assertAlmostEqual(metrics["shape_score"], 1)
def test_shape_similarity(self):
"""
Verify that the shape similarity measure is producing sane outputs.
"""
# TODO: Get deepcopy working for ProjectScene and make a
# simpler example for faster unit test.
evaluator = VoxelEvaluator(self.submission, self.ground_truth,
self.settings)
obj1 = self.submission.elements["1069"]
obj2 = self.ground_truth.elements["1069"]
# verify no offset gives sim = 1
sim = evaluator._shape_similarity(obj1, obj2)
self.assertAlmostEqual(sim, 1)
# verify small offset gives sim between 0 and 1
voxel_centers_orig = obj2.voxel_centers
obj2.voxel_centers = obj2.voxel_centers + np.array([0.2, 0, 0])
sim = evaluator._shape_similarity(obj1, obj2)
self.assertTrue(sim < 1 and sim > 0)
# verify large offset gives sim = 0
obj2.voxel_centers = obj2.voxel_centers + np.array([1, 0, 0])
sim = evaluator._shape_similarity(obj1, obj2)
self.assertAlmostEqual(sim, 0)
obj2.voxel_centers = voxel_centers_orig
shape_score = evaluator.shape_score()
self.assertAlmostEqual(shape_score, 1)
def test_rms_color_error(self):
evaluator = VoxelEvaluator(self.submission, self.ground_truth,
self.settings)
rms_color_error = evaluator.rms_color_error()
self.assertTrue(rms_color_error < 25)
def test_semantics_score(self):
evaluator = VoxelEvaluator(self.submission, self.ground_truth,
self.settings)
semantics_score = evaluator.semantics_score()
self.assertAlmostEqual(semantics_score, 1)