def test_matches_too_close_are_trivial_matches(self): trajectory = { 1.33333: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 3.66667: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 14.33333: tf.Transform(location=(-100, 100, 0), rotation=(0, 0, 0, 1)), 15.66667: tf.Transform(location=(-100, 100, 0), rotation=(0, 0, 0, 1)) } closures = {3.66667: 1.33333} trial_result = MockTrialResult(gt_trajectory=trajectory, loop_closures=closures) # Perform the benchmark with a larger threshold benchmark = lc.BenchmarkLoopClosure(distance_threshold=20, trivial_closure_index_distance=10) result = benchmark.benchmark_results(trial_result) self.assertEqual(match_res.MatchType.FALSE_POSITIVE, result.matches[3.66667]) self.assertEqual(match_res.MatchType.TRUE_NEGATIVE, result.matches[15.66667]) # Try again with a smaller threshold, should become a true positive, # since the indexes are further appart than the threshold benchmark = lc.BenchmarkLoopClosure(distance_threshold=20, trivial_closure_index_distance=1) result = benchmark.benchmark_results(trial_result) self.assertEqual(match_res.MatchType.TRUE_POSITIVE, result.matches[3.66667]) self.assertEqual(match_res.MatchType.FALSE_NEGATIVE, result.matches[15.66667])
def test_distance_threshold_determines_acceptable_matches(self): trajectory = { 1.33333: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 10.66667: tf.Transform(location=(110, 100, 0), rotation=(0, 0, 0, 1)), 15.33333: tf.Transform(location=(-100, 100, 0), rotation=(0, 0, 0, 1)), 20.66667: tf.Transform(location=(-110, 100, 0), rotation=(0, 0, 0, 1)) } closures = {10.66667: 1.33333} trial_result = MockTrialResult(gt_trajectory=trajectory, loop_closures=closures) # Perform the benchmark with a larger threshold benchmark = lc.BenchmarkLoopClosure(distance_threshold=20) result = benchmark.benchmark_results(trial_result) self.assertEqual(match_res.MatchType.TRUE_POSITIVE, result.matches[10.66667]) self.assertEqual(match_res.MatchType.FALSE_NEGATIVE, result.matches[20.66667]) # Try again with a smaller threshold, should become a false positive benchmark = lc.BenchmarkLoopClosure(distance_threshold=1) result = benchmark.benchmark_results(trial_result) self.assertEqual(match_res.MatchType.FALSE_POSITIVE, result.matches[10.66667]) self.assertEqual(match_res.MatchType.TRUE_NEGATIVE, result.matches[20.66667])
def test_benchmark_produces_expected_results(self): # Perform the benchmark benchmark = tracking.TrackingBenchmark() result = benchmark.benchmark_results(self.trial_result) self.assertNotIsInstance(result, core.benchmark.FailedBenchmark) # The expectations are based on the trajectory and tracking stats in setUp self.assertEqual(4, result.times_lost) self.assertEqual(1.3333, result.lost_intervals[0].start_time) self.assertEqual(1.6667, result.lost_intervals[0].end_time) self.assertEqual(1.6667 - 1.3333, result.lost_intervals[0].duration) self.assertEqual(10, result.lost_intervals[0].distance) self.assertEqual(1, result.lost_intervals[0].frames) self.assertEqual(2.3333, result.lost_intervals[1].start_time) self.assertEqual(3, result.lost_intervals[1].end_time) self.assertEqual(3 - 2.3333, result.lost_intervals[1].duration) self.assertEqual(20, result.lost_intervals[1].distance) self.assertEqual(2, result.lost_intervals[1].frames) self.assertEqual(3.3333, result.lost_intervals[2].start_time) self.assertEqual(4.3333, result.lost_intervals[2].end_time) self.assertAlmostEqual(1, result.lost_intervals[2].duration) self.assertEqual(30, result.lost_intervals[2].distance) self.assertEqual(3, result.lost_intervals[2].frames) self.assertEqual(4.6667, result.lost_intervals[3].start_time) self.assertEqual(4.6667, result.lost_intervals[3].end_time) self.assertEqual(0, result.lost_intervals[3].duration) self.assertEqual(0, result.lost_intervals[3].distance) self.assertEqual(1, result.lost_intervals[3].frames)
def test_benchmark_results_returns_a_benchmark_result(self): benchmark = tracking.TrackingBenchmark() result = benchmark.benchmark_results(self.trial_result) self.assertIsInstance(result, core.benchmark.BenchmarkResult) self.assertNotIsInstance(result, core.benchmark.FailedBenchmark) self.assertEqual(benchmark.identifier, result.benchmark) self.assertEqual(self.trial_result.identifier, result.trial_result)
def test_benchmark_accepts_any_of_multiple_closures(self): # Create a trial result with multiple valid closures trajectory = { 1.33333: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 5.33333: tf.Transform(location=(-100, -100, 0), rotation=(0, 0, 0, 1)), 10.66667: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 12.33333: tf.Transform(location=(-100, -100, 0), rotation=(0, 0, 0, 1)), 15: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 17.66667: tf.Transform(location=(-100, -100, 0), rotation=(0, 0, 0, 1)), 20.66667: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)) } trial_result = MockTrialResult(gt_trajectory=trajectory, loop_closures={}) benchmark = lc.BenchmarkLoopClosure(distance_threshold=20) # Simple cartesian product to get all the different combinations of closures for the location group1 = [20.66667, 15, 10.66667, 1.33333] valid_pairs = [(idx, closure) for idx in group1 for closure in group1 if closure < idx] # Test different possible closures for idx, closure in valid_pairs: trial_result.loop_closures = {idx: closure} result = benchmark.benchmark_results(trial_result) self.assertEqual(match_res.MatchType.TRUE_POSITIVE, result.matches[idx])
def test_offset_shifts_query_trajectory_time(self): # Create a new noise trajectory with a large time offset comp_traj, noise = create_noise( self.trial_result.ground_truth_trajectory, self.random, time_offset=1000) self.trial_result.computed_trajectory = comp_traj # This should fail due to the offset benchmark = rpe.BenchmarkRPE() result = benchmark.benchmark_results(self.trial_result) self.assertIsInstance(result, core.benchmark.FailedBenchmark) # This one should work, since the offset brings things back close together benchmark.offset = -1000 result = benchmark.benchmark_results(self.trial_result) self.assertNotIsInstance(result, core.benchmark.FailedBenchmark)
def test_scale_affects_trajectory_position(self): # Manually scale the computed trajectory scale = 4243 self.trial_result.computed_trajectory = {} for key, pose in self.trial_result.ground_truth_trajectory.items(): self.trial_result.computed_trajectory[key] = tf.Transform( location=pose.location / scale, rotation=pose.rotation_quat(True), w_first=True) # This should have a large error due to the bad scale benchmark = rpe.BenchmarkRPE() unscaled_result = benchmark.benchmark_results(self.trial_result) # This one should have a more reasonable error benchmark.scale = scale result = benchmark.benchmark_results(self.trial_result) self.assertLess(result.trans_max, unscaled_result.trans_max)
def test_benchmark_results_returns_a_benchmark_result(self): trial_result = MockTrialResult( gt_bboxes={oid.ObjectId(): [bbox_trial.BoundingBox(('cup',), 0.8256, 15, 22, 100, 100)]}, bboxes={oid.ObjectId(): [bbox_trial.BoundingBox(('cup',), 0.8256, 15, 22, 100, 100)]}) benchmark = bbox_overlap.BoundingBoxOverlapBenchmark() result = benchmark.benchmark_results(trial_result) self.assertIsInstance(result, core.benchmark.BenchmarkResult) self.assertNotIsInstance(result, core.benchmark.FailedBenchmark) self.assertIsInstance(result, bbox_result.BoundingBoxOverlapBenchmarkResult) self.assertEqual(benchmark.identifier, result.benchmark) self.assertEqual(trial_result.identifier, result.trial_result)
def test_benchmark_results_fails_for_no_matching_timestaps(self): # Create a new computed trajectory with no matching timestamps self.trial_result.computed_trajectory = { time + 10000: pose for time, pose in self.trial_result.ground_truth_trajectory.items() } # Perform the benchmark benchmark = rpe.BenchmarkRPE() result = benchmark.benchmark_results(self.trial_result) self.assertIsInstance(result, core.benchmark.FailedBenchmark)
def test_benchmark_measures_match_for_all_stamps(self): random = np.random.RandomState(1563) trajectory = { random.uniform(0, 600): tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)) for _ in range(100) } trial_result = MockTrialResult(gt_trajectory=trajectory, loop_closures={}) benchmark = lc.BenchmarkLoopClosure(distance_threshold=20) result = benchmark.benchmark_results(trial_result) for stamp in trajectory.keys(): self.assertIn(stamp, result.matches)
def test_benchmark_results_estimates_no_error_for_identical_trajectory( self): # Copy the ground truth exactly self.trial_result.computed_trajectory = copy.deepcopy( self.trial_result.ground_truth_trajectory) benchmark = rpe.BenchmarkRPE() result = benchmark.benchmark_results(self.trial_result) if isinstance(result, core.benchmark.FailedBenchmark): print(result.reason) for time, error in result.translational_error.items(): self.assertAlmostEqual(0, error) for time, error in result.rotational_error.items(): self.assertAlmostEqual(0, error)
def test_benchmark_results_returns_a_benchmark_result(self): trial_result = MockTrialResult(gt_trajectory={ 1.33333: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 10.66667: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)) }, loop_closures={}) benchmark = lc.BenchmarkLoopClosure(distance_threshold=20) result = benchmark.benchmark_results(trial_result) self.assertIsInstance(result, core.benchmark.BenchmarkResult) self.assertNotIsInstance(result, core.benchmark.FailedBenchmark) self.assertIsInstance(result, match_res.MatchBenchmarkResult) self.assertEqual(benchmark.identifier, result.benchmark) self.assertEqual(trial_result.identifier, result.trial_result)
def test_benchmark_detects_false_negative(self): # Create a trial result with a correct loop closure trajectory = { 1.33333: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)), 10.66667: tf.Transform(location=(100, 100, 0), rotation=(0, 0, 0, 1)) } closures = {} trial_result = MockTrialResult(gt_trajectory=trajectory, loop_closures=closures) # Perform the benchmark benchmark = lc.BenchmarkLoopClosure(distance_threshold=20) result = benchmark.benchmark_results(trial_result) self.assertEqual(match_res.MatchType.FALSE_NEGATIVE, result.matches[10.66667])
def test_benchmark_results_estimates_no_error_for_noiseless_trajectory( self): # Create a new computed trajectory with no noise comp_traj, noise = create_noise( self.trial_result.ground_truth_trajectory, self.random, time_offset=0, time_noise=0, loc_noise=0, rot_noise=0) self.trial_result.computed_trajectory = comp_traj benchmark = rpe.BenchmarkRPE() result = benchmark.benchmark_results(self.trial_result) for time, error in result.translational_error.items(): self.assertAlmostEqual(0, error) for time, error in result.rotational_error.items(): self.assertAlmostEqual(0, error)
def test_benchmark_matches_each_gt_box_only_once(self): id1 = oid.ObjectId() trial_result = MockTrialResult( gt_bboxes={ id1: [bbox_trial.BoundingBox({'cup'}, 1, 15, 22, 175, 175)] }, bboxes={ id1: [bbox_trial.BoundingBox({'cup'}, 1, 15, 22, 100, 100), bbox_trial.BoundingBox({'cup'}, 1, 115, 122, 50, 50), bbox_trial.BoundingBox({'cup'}, 1, 165, 172, 25, 25)], } ) benchmark = bbox_overlap.BoundingBoxOverlapBenchmark() result = benchmark.benchmark_results(trial_result) self.assertIn(id1, result.overlaps) self.assertEqual(3, len(result.overlaps[id1])) self.assertEqual({ 'overlap': 10000, 'bounding_box_area': 10000, 'ground_truth_area': 30625, 'confidence': 1.0, 'bounding_box_classes': ('cup',), 'ground_truth_classes': ('cup',) }, result.overlaps[id1][0]) self.assertEqual({ 'overlap': 0, 'bounding_box_area': 2500, 'ground_truth_area': 0, 'confidence': 1.0, 'bounding_box_classes': ('cup',), 'ground_truth_classes': tuple() }, result.overlaps[id1][1]) self.assertEqual({ 'overlap': 0, 'bounding_box_area': 625, 'ground_truth_area': 0, 'confidence': 1.0, 'bounding_box_classes': ('cup',), 'ground_truth_classes': tuple() }, result.overlaps[id1][2])
def benchmark_results(benchmark, database_client, config=None, trained_state_id=None): if (not isinstance(benchmark, core.benchmark.Benchmark) or not isinstance(database_client, database.client.DatabaseClient)): return if config is None: config = {} else: config = dict(config) config = du.defaults(config, {}) existing_results_query = {'benchmark': benchmark.identifier} if trained_state_id is not None: existing_results_query['trained_state'] = trained_state_id existing_results = database_client.results_collection.find( existing_results_query, { '_id': False, 'trial_result': True }) existing_results = [result['trial_result'] for result in existing_results] trial_results = database_client.trials_collection.find( du.defaults(benchmark.get_benchmark_requirements(), {'_id': { '$nin': existing_results }})) for s_trial_result in trial_results: trial_result = database_client.deserialize_entity(s_trial_result) s_dataset = database_client.dataset_collection.find_one( {'_id': trial_result.image_source_id}) dataset = database_client.deserialize_entity(s_dataset) dataset_images = dataset.load_images(database_client) benchmark_result = benchmark.benchmark_results(dataset_images, trial_result) database_client.results_collection.insert(benchmark_result.serialize())
def test_benchmark_measures_score_per_gt_bounding_box(self): id1 = oid.ObjectId() id2 = oid.ObjectId() id3 = oid.ObjectId() id4 = oid.ObjectId() trial_result = MockTrialResult( gt_bboxes={ id1: [bbox_trial.BoundingBox({'cup'}, 1, 15, 22, 100, 100)], id2: [bbox_trial.BoundingBox({'car'}, 1, 15, 22, 100, 100)], id3: [bbox_trial.BoundingBox({'cow'}, 1, 15, 22, 100, 100)], id4: [bbox_trial.BoundingBox({'cat'}, 1, 15, 22, 100, 100)] }, bboxes={ id1: [bbox_trial.BoundingBox({'cup'}, 1, 15, 22, 100, 100)], # Matched exactly id2: [bbox_trial.BoundingBox({'car'}, 0.8256, 15, 22, 100, 100)], # Only confidence reduced id3: [bbox_trial.BoundingBox({'cow'}, 1, 25, 32, 95, 95)], # Slightly misplaced id4: [bbox_trial.BoundingBox({'cat'}, 0.75, 25, 32, 95, 95)] # Reduced confidence and slightly misplaced } ) benchmark = bbox_overlap.BoundingBoxOverlapBenchmark() result = benchmark.benchmark_results(trial_result) self.assertIn(id1, result.overlaps) self.assertIn(id2, result.overlaps) self.assertIn(id3, result.overlaps) self.assertIn(id4, result.overlaps) self.assertEqual(1, len(result.overlaps[id1])) self.assertEqual(1, len(result.overlaps[id2])) self.assertEqual(1, len(result.overlaps[id3])) self.assertEqual(1, len(result.overlaps[id4])) self.assertEqual({ 'overlap': 10000, 'bounding_box_area': 10000, 'ground_truth_area': 10000, 'confidence': 1.0, 'bounding_box_classes': ('cup',), 'ground_truth_classes': ('cup',) }, result.overlaps[id1][0]) self.assertEqual({ 'overlap': 10000, 'bounding_box_area': 10000, 'ground_truth_area': 10000, 'confidence': 0.8256, 'bounding_box_classes': ('car',), 'ground_truth_classes': ('car',) }, result.overlaps[id2][0]) self.assertEqual({ 'overlap': 8100, 'bounding_box_area': 9025, 'ground_truth_area': 10000, 'confidence': 1.0, 'bounding_box_classes': ('cow',), 'ground_truth_classes': ('cow',) }, result.overlaps[id3][0]) self.assertEqual({ 'overlap': 8100, 'bounding_box_area': 9025, 'ground_truth_area': 10000, 'confidence': 0.75, 'bounding_box_classes': ('cat',), 'ground_truth_classes': ('cat',) }, result.overlaps[id4][0])
def test_benchmark_results_estimates_reasonable_trajectory_noise(self): benchmark = rpe.BenchmarkRPE() result = benchmark.benchmark_results(self.trial_result)