def write_statistics_to_file(output_folder: str, labels: List[str], results: List[List[Tuple[str, float, float]]], bins_as_str: List[str], force: bool) -> None: """ Writes evaluation statistics to text files. Results and labels must be synchronised. :param output_folder: full path of folder to write statistics files in. :param labels: labels for the statistics files :param results: results to compute statistics :param bins_as_str: list of bin names :param force: Silently overwrite results files if already exists. """ full_path = path.join(output_folder, STATISTICS_FILENAME + '.txt') safe_remove_file(full_path, force) bins = [(float(split_bin[0]), float(split_bin[1])) for split_bin in map(lambda x: x.split(), bins_as_str)] print_line = '' for i in range(0, len(results)): label = labels[i] result = results[i] number_of_images = len(result) positions_errors_all = [ position_error if not isnan(position_error) else float("inf") for name, position_error, rotation_error in result ] rotation_errors_all = [ rotation_error if not isnan(rotation_error) else float("inf") for name, position_error, rotation_error in result ] positions_errors = [ position_error for name, position_error, rotation_error in result if not isnan(position_error) ] rotation_errors = [ rotation_error for name, position_error, rotation_error in result if not isnan(rotation_error) ] print_line += 'Model: {}\n\n'.format(label) print_line += 'Found {} / {} image positions ({:.2f} %).\n'.format( len(positions_errors), number_of_images, float(100.0 * len(positions_errors) / number_of_images)) print_line += 'Found {} / {} image rotations ({:.2f} %).\n'.format( len(rotation_errors), number_of_images, float(100.0 * len(rotation_errors) / number_of_images)) print_line += 'Localized images: mean=({:.4f}m, {:.4f} deg) / median=({:.4f}m, {:.4f} deg)\n'.format( mean(positions_errors), mean(rotation_errors), median(positions_errors), median(rotation_errors)) print_line += 'All: median=({:.4f}m, {:.4f} deg)\n'.format( median(positions_errors_all), median(rotation_errors_all)) print_line += 'Min: {:.4f}m; {:.4f} deg\n'.format( min(positions_errors), min(rotation_errors)) print_line += 'Max: {:.4f}m; {:.4f} deg\n\n'.format( max(positions_errors), max(rotation_errors)) filled_bins = fill_bins(result, bins) bins_lines = [ '({}m, {} deg): {:.2f}%\n'.format( position_error, rotation_error, (number_of_images_in_bin / number_of_images) * 100.0) for position_error, rotation_error, number_of_images_in_bin in filled_bins ] print_line += ''.join(bins_lines) print_line += '\n' print(print_line) with open(full_path, 'w') as fid: fid.write(print_line)
def test_evaluation(self): position = [1.658, 0, 0] position_a = [2.658, 0, 0] position_b = [1.758, 0, 0] position_c = [10.1, 0, 0] position_d = [2., 0, 0] position_e = [6.658, 0, 0] rotation = quaternion.from_euler_angles(np.deg2rad(110.0), 0, 0) rotation_a = quaternion.from_euler_angles(np.deg2rad(111.0), 0, 0) rotation_b = quaternion.from_euler_angles(np.deg2rad(108.0), 0, 0) rotation_c = quaternion.from_euler_angles(np.deg2rad(10.0), 0, 0) rotation_d = quaternion.from_euler_angles(np.deg2rad(110.0), 0, 0) pose_gt = kapture.PoseTransform(r=rotation, t=position).inverse() pose_a = kapture.PoseTransform(r=rotation_a, t=position_a).inverse() pose_b = kapture.PoseTransform(r=rotation_b, t=position_b).inverse() pose_c = kapture.PoseTransform(r=rotation_c, t=position_c).inverse() pose_d = kapture.PoseTransform(r=rotation_d, t=position_d).inverse() pose_e = kapture.PoseTransform(r=None, t=[-x for x in position_e]) kdata = kapture.Kapture(sensors=kapture.Sensors(), records_camera=kapture.RecordsCamera(), trajectories=kapture.Trajectories()) kdata.sensors['cam0'] = kapture.Camera( kapture.CameraType.UNKNOWN_CAMERA, [25, 13]) kdata.records_camera[(0, 'cam0')] = 'a' kdata.records_camera[(1, 'cam0')] = 'b' kdata.records_camera[(2, 'cam0')] = 'c' kdata.records_camera[(3, 'cam0')] = 'd' kdata.records_camera[(4, 'cam0')] = 'e' kdata.trajectories[(0, 'cam0')] = pose_a kdata.trajectories[(1, 'cam0')] = pose_b kdata.trajectories[(2, 'cam0')] = pose_c kdata.trajectories[(3, 'cam0')] = pose_d kdata2 = copy.deepcopy(kdata) kdata2.trajectories[(4, 'cam0')] = pose_e kdata2.records_camera[(5, 'cam0')] = 'f' kdata_gt = copy.deepcopy(kdata2) kdata_gt.trajectories[(0, 'cam0')] = pose_gt kdata_gt.trajectories[(1, 'cam0')] = pose_gt kdata_gt.trajectories[(2, 'cam0')] = pose_gt kdata_gt.trajectories[(3, 'cam0')] = pose_gt kdata_gt.trajectories[(4, 'cam0')] = pose_gt kdata_gt.trajectories[(5, 'cam0')] = pose_gt kdata_list = [kdata, kdata2, kdata_gt] intersection = {'a', 'b', 'c', 'd', 'e'} result1 = evaluate(kdata, kdata_gt, intersection) self.assertEqual(len(result1), 5) self.assertEqual(result1[0][0], 'a') self.assertAlmostEqual(result1[0][1], 1.0) self.assertAlmostEqual(result1[0][2], 1.0) self.assertEqual(result1[1][0], 'b') self.assertAlmostEqual(result1[1][1], 0.1) self.assertAlmostEqual(result1[1][2], 2.0) self.assertEqual(result1[2][0], 'c') self.assertAlmostEqual(result1[2][1], 8.442) self.assertAlmostEqual(result1[2][2], 100.0) self.assertEqual(result1[3][0], 'd') self.assertAlmostEqual(result1[3][1], 0.342) self.assertAlmostEqual(result1[3][2], 0.0) self.assertEqual(result1[4][0], 'e') self.assertTrue(math.isnan(result1[4][1])) self.assertTrue(math.isnan(result1[4][2])) result2 = evaluate(kdata2, kdata_gt, intersection) self.assertEqual(len(result2), 5) self.assertEqual(result2[0][0], 'a') self.assertAlmostEqual(result2[0][1], 1.0) self.assertAlmostEqual(result2[0][2], 1.0) self.assertEqual(result2[1][0], 'b') self.assertAlmostEqual(result2[1][1], 0.1) self.assertAlmostEqual(result2[1][2], 2.0) self.assertEqual(result2[2][0], 'c') self.assertAlmostEqual(result2[2][1], 8.442) self.assertAlmostEqual(result2[2][2], 100.0) self.assertEqual(result2[3][0], 'd') self.assertAlmostEqual(result2[3][1], 0.342) self.assertAlmostEqual(result2[3][2], 0.0) self.assertEqual(result2[4][0], 'e') self.assertAlmostEqual(result2[4][1], 5.0) self.assertTrue(math.isnan(result2[4][2])) bins1 = fill_bins(result1, [(0.9, 5), (10, 105)]) self.assertEqual(len(bins1), 2) self.assertEqual(bins1[0][0], 0.9) self.assertEqual(bins1[0][1], 5) self.assertEqual(bins1[0][2], 2) self.assertEqual(bins1[1][0], 10) self.assertEqual(bins1[1][1], 105) self.assertEqual(bins1[1][2], 4) bins2 = fill_bins(result1, [(0.9, 5), (10, 105)]) self.assertEqual(len(bins2), 2) self.assertEqual(bins2[0][0], 0.9) self.assertEqual(bins2[0][1], 5) self.assertEqual(bins2[0][2], 2) self.assertEqual(bins2[1][0], 10) self.assertEqual(bins2[1][1], 105) self.assertEqual(bins2[1][2], 4) bins3 = fill_bins(result2, [(0.9, math.nan), (10, math.nan)]) self.assertEqual(len(bins3), 2) self.assertEqual(bins3[0][0], 0.9) self.assertTrue(math.isnan(bins3[0][1])) self.assertEqual(bins3[0][2], 2) self.assertEqual(bins3[1][0], 10) self.assertTrue(math.isnan(bins3[1][1])) self.assertEqual(bins3[1][2], 5) bins4 = fill_bins(result2, [(0.9, -1), (10, -1)]) self.assertEqual(len(bins4), 2) self.assertEqual(bins4[0][0], 0.9) self.assertEqual(bins4[0][1], -1) self.assertEqual(bins4[0][2], 2) self.assertEqual(bins4[1][0], 10) self.assertEqual(bins4[1][1], -1) self.assertEqual(bins4[1][2], 5)
def plot_localized_over_position_threshold(output_folder: str, labels: List[str], results: List[List[Tuple[str, float, float]]], rotation_threshold: float, plot_max: int, title: str, plot_loc: str, plot_font_size: int, plot_legend_font_size: int, force: bool) -> None: """ Plot image with localized positions. :param output_folder: full path of folder to write plot files in. :param labels: labels for the plot images :param results: results to compute the images to plot :param rotation_threshold: rotation threshold :param plot_max: max points to plot :param title: title for the plotted image :param plot_loc: location for the legend :param plot_font_size: font size to plot with :param plot_legend_font_size: font size for the legend :param force: Silently overwrite plot files if already exists. """ import matplotlib.pylab as plt plt.rcParams['font.size'] = plot_font_size plt.rcParams['legend.fontsize'] = plot_legend_font_size position_thresholds = np.linspace( 0.0, 1.0, num=101, dtype=np.float64) * plot_max # thresholds in cm bins = [(position_threshold / 100.0, rotation_threshold) for position_threshold in position_thresholds ] # convert to thresholds in m number_of_images = None for i in range(0, len(results)): label = labels[i] result = results[i] if number_of_images is None: number_of_images = len(result) else: # just make sure we are comparing comparable things assert (number_of_images == len(result)) filled_bins = [(t[2] / number_of_images) * 100.0 for t in fill_bins(result, bins)] # convert back to cm plt.plot(position_thresholds, filled_bins, lw=2, label=label) plt.xlabel('position error threshold (cm)') plt.ylabel('localized images (%)') plt.ylim([0, 100]) # 0 to 100 % plt.legend(loc=plot_loc) plt.xlim([position_thresholds[0], position_thresholds[-1]]) # plot grid grid_step = int(plot_max / 10.0) for i in range(grid_step, plot_max - grid_step + 1, grid_step): plt.plot([i, i], [0, 100], 'k', alpha=0.1) for i in range(10, 91, 10): plt.plot([0, plot_max], [i, i], 'k', alpha=0.1) plot_title = title if rotation_threshold >= 0: plot_title += ( ' ' if title else '') + r'$\alpha_{th}$' + '={} deg'.format(rotation_threshold) plt.title(plot_title) full_path = path.join(output_folder, PLOT_FILENAME + '.png') safe_remove_file(full_path, force) plt.savefig(full_path, bbox_inches='tight')