def write_statistics_to_file(output_folder: str, labels: List[str],
results: List[List[Tuple[str, float, float]]],
bins_as_str: List[str], force: bool) -> None:
"""
Writes evaluation statistics to text files. Results and labels must be synchronised.
:param output_folder: full path of folder to write statistics files in.
:param labels: labels for the statistics files
:param results: results to compute statistics
:param bins_as_str: list of bin names
:param force: Silently overwrite results files if already exists.
"""
full_path = path.join(output_folder, STATISTICS_FILENAME + '.txt')
safe_remove_file(full_path, force)
bins = [(float(split_bin[0]), float(split_bin[1]))
for split_bin in map(lambda x: x.split(), bins_as_str)]
print_line = ''
for i in range(0, len(results)):
label = labels[i]
result = results[i]
number_of_images = len(result)
positions_errors_all = [
position_error if not isnan(position_error) else float("inf")
for name, position_error, rotation_error in result
]
rotation_errors_all = [
rotation_error if not isnan(rotation_error) else float("inf")
for name, position_error, rotation_error in result
]
positions_errors = [
position_error for name, position_error, rotation_error in result
if not isnan(position_error)
]
rotation_errors = [
rotation_error for name, position_error, rotation_error in result
if not isnan(rotation_error)
]
print_line += 'Model: {}\n\n'.format(label)
print_line += 'Found {} / {} image positions ({:.2f} %).\n'.format(
len(positions_errors), number_of_images,
float(100.0 * len(positions_errors) / number_of_images))
print_line += 'Found {} / {} image rotations ({:.2f} %).\n'.format(
len(rotation_errors), number_of_images,
float(100.0 * len(rotation_errors) / number_of_images))
print_line += 'Localized images: mean=({:.4f}m, {:.4f} deg) / median=({:.4f}m, {:.4f} deg)\n'.format(
mean(positions_errors), mean(rotation_errors),
median(positions_errors), median(rotation_errors))
print_line += 'All: median=({:.4f}m, {:.4f} deg)\n'.format(
median(positions_errors_all), median(rotation_errors_all))
print_line += 'Min: {:.4f}m; {:.4f} deg\n'.format(
min(positions_errors), min(rotation_errors))
print_line += 'Max: {:.4f}m; {:.4f} deg\n\n'.format(
max(positions_errors), max(rotation_errors))
filled_bins = fill_bins(result, bins)
bins_lines = [
'({}m, {} deg): {:.2f}%\n'.format(
position_error, rotation_error,
(number_of_images_in_bin / number_of_images) * 100.0)
for position_error, rotation_error, number_of_images_in_bin in
filled_bins
]
print_line += ''.join(bins_lines)
print_line += '\n'
print(print_line)
with open(full_path, 'w') as fid:
fid.write(print_line)
def test_evaluation(self):
position = [1.658, 0, 0]
position_a = [2.658, 0, 0]
position_b = [1.758, 0, 0]
position_c = [10.1, 0, 0]
position_d = [2., 0, 0]
position_e = [6.658, 0, 0]
rotation = quaternion.from_euler_angles(np.deg2rad(110.0), 0, 0)
rotation_a = quaternion.from_euler_angles(np.deg2rad(111.0), 0, 0)
rotation_b = quaternion.from_euler_angles(np.deg2rad(108.0), 0, 0)
rotation_c = quaternion.from_euler_angles(np.deg2rad(10.0), 0, 0)
rotation_d = quaternion.from_euler_angles(np.deg2rad(110.0), 0, 0)
pose_gt = kapture.PoseTransform(r=rotation, t=position).inverse()
pose_a = kapture.PoseTransform(r=rotation_a, t=position_a).inverse()
pose_b = kapture.PoseTransform(r=rotation_b, t=position_b).inverse()
pose_c = kapture.PoseTransform(r=rotation_c, t=position_c).inverse()
pose_d = kapture.PoseTransform(r=rotation_d, t=position_d).inverse()
pose_e = kapture.PoseTransform(r=None, t=[-x for x in position_e])
kdata = kapture.Kapture(sensors=kapture.Sensors(),
records_camera=kapture.RecordsCamera(),
trajectories=kapture.Trajectories())
kdata.sensors['cam0'] = kapture.Camera(
kapture.CameraType.UNKNOWN_CAMERA, [25, 13])
kdata.records_camera[(0, 'cam0')] = 'a'
kdata.records_camera[(1, 'cam0')] = 'b'
kdata.records_camera[(2, 'cam0')] = 'c'
kdata.records_camera[(3, 'cam0')] = 'd'
kdata.records_camera[(4, 'cam0')] = 'e'
kdata.trajectories[(0, 'cam0')] = pose_a
kdata.trajectories[(1, 'cam0')] = pose_b
kdata.trajectories[(2, 'cam0')] = pose_c
kdata.trajectories[(3, 'cam0')] = pose_d
kdata2 = copy.deepcopy(kdata)
kdata2.trajectories[(4, 'cam0')] = pose_e
kdata2.records_camera[(5, 'cam0')] = 'f'
kdata_gt = copy.deepcopy(kdata2)
kdata_gt.trajectories[(0, 'cam0')] = pose_gt
kdata_gt.trajectories[(1, 'cam0')] = pose_gt
kdata_gt.trajectories[(2, 'cam0')] = pose_gt
kdata_gt.trajectories[(3, 'cam0')] = pose_gt
kdata_gt.trajectories[(4, 'cam0')] = pose_gt
kdata_gt.trajectories[(5, 'cam0')] = pose_gt
kdata_list = [kdata, kdata2, kdata_gt]
intersection = {'a', 'b', 'c', 'd', 'e'}
result1 = evaluate(kdata, kdata_gt, intersection)
self.assertEqual(len(result1), 5)
self.assertEqual(result1[0][0], 'a')
self.assertAlmostEqual(result1[0][1], 1.0)
self.assertAlmostEqual(result1[0][2], 1.0)
self.assertEqual(result1[1][0], 'b')
self.assertAlmostEqual(result1[1][1], 0.1)
self.assertAlmostEqual(result1[1][2], 2.0)
self.assertEqual(result1[2][0], 'c')
self.assertAlmostEqual(result1[2][1], 8.442)
self.assertAlmostEqual(result1[2][2], 100.0)
self.assertEqual(result1[3][0], 'd')
self.assertAlmostEqual(result1[3][1], 0.342)
self.assertAlmostEqual(result1[3][2], 0.0)
self.assertEqual(result1[4][0], 'e')
self.assertTrue(math.isnan(result1[4][1]))
self.assertTrue(math.isnan(result1[4][2]))
result2 = evaluate(kdata2, kdata_gt, intersection)
self.assertEqual(len(result2), 5)
self.assertEqual(result2[0][0], 'a')
self.assertAlmostEqual(result2[0][1], 1.0)
self.assertAlmostEqual(result2[0][2], 1.0)
self.assertEqual(result2[1][0], 'b')
self.assertAlmostEqual(result2[1][1], 0.1)
self.assertAlmostEqual(result2[1][2], 2.0)
self.assertEqual(result2[2][0], 'c')
self.assertAlmostEqual(result2[2][1], 8.442)
self.assertAlmostEqual(result2[2][2], 100.0)
self.assertEqual(result2[3][0], 'd')
self.assertAlmostEqual(result2[3][1], 0.342)
self.assertAlmostEqual(result2[3][2], 0.0)
self.assertEqual(result2[4][0], 'e')
self.assertAlmostEqual(result2[4][1], 5.0)
self.assertTrue(math.isnan(result2[4][2]))
bins1 = fill_bins(result1, [(0.9, 5), (10, 105)])
self.assertEqual(len(bins1), 2)
self.assertEqual(bins1[0][0], 0.9)
self.assertEqual(bins1[0][1], 5)
self.assertEqual(bins1[0][2], 2)
self.assertEqual(bins1[1][0], 10)
self.assertEqual(bins1[1][1], 105)
self.assertEqual(bins1[1][2], 4)
bins2 = fill_bins(result1, [(0.9, 5), (10, 105)])
self.assertEqual(len(bins2), 2)
self.assertEqual(bins2[0][0], 0.9)
self.assertEqual(bins2[0][1], 5)
self.assertEqual(bins2[0][2], 2)
self.assertEqual(bins2[1][0], 10)
self.assertEqual(bins2[1][1], 105)
self.assertEqual(bins2[1][2], 4)
bins3 = fill_bins(result2, [(0.9, math.nan), (10, math.nan)])
self.assertEqual(len(bins3), 2)
self.assertEqual(bins3[0][0], 0.9)
self.assertTrue(math.isnan(bins3[0][1]))
self.assertEqual(bins3[0][2], 2)
self.assertEqual(bins3[1][0], 10)
self.assertTrue(math.isnan(bins3[1][1]))
self.assertEqual(bins3[1][2], 5)
bins4 = fill_bins(result2, [(0.9, -1), (10, -1)])
self.assertEqual(len(bins4), 2)
self.assertEqual(bins4[0][0], 0.9)
self.assertEqual(bins4[0][1], -1)
self.assertEqual(bins4[0][2], 2)
self.assertEqual(bins4[1][0], 10)
self.assertEqual(bins4[1][1], -1)
self.assertEqual(bins4[1][2], 5)
def plot_localized_over_position_threshold(output_folder: str,
labels: List[str],
results: List[List[Tuple[str, float,
float]]],
rotation_threshold: float,
plot_max: int, title: str,
plot_loc: str, plot_font_size: int,
plot_legend_font_size: int,
force: bool) -> None:
"""
Plot image with localized positions.
:param output_folder: full path of folder to write plot files in.
:param labels: labels for the plot images
:param results: results to compute the images to plot
:param rotation_threshold: rotation threshold
:param plot_max: max points to plot
:param title: title for the plotted image
:param plot_loc: location for the legend
:param plot_font_size: font size to plot with
:param plot_legend_font_size: font size for the legend
:param force: Silently overwrite plot files if already exists.
"""
import matplotlib.pylab as plt
plt.rcParams['font.size'] = plot_font_size
plt.rcParams['legend.fontsize'] = plot_legend_font_size
position_thresholds = np.linspace(
0.0, 1.0, num=101, dtype=np.float64) * plot_max # thresholds in cm
bins = [(position_threshold / 100.0, rotation_threshold)
for position_threshold in position_thresholds
] # convert to thresholds in m
number_of_images = None
for i in range(0, len(results)):
label = labels[i]
result = results[i]
if number_of_images is None:
number_of_images = len(result)
else:
# just make sure we are comparing comparable things
assert (number_of_images == len(result))
filled_bins = [(t[2] / number_of_images) * 100.0
for t in fill_bins(result, bins)] # convert back to cm
plt.plot(position_thresholds, filled_bins, lw=2, label=label)
plt.xlabel('position error threshold (cm)')
plt.ylabel('localized images (%)')
plt.ylim([0, 100]) # 0 to 100 %
plt.legend(loc=plot_loc)
plt.xlim([position_thresholds[0], position_thresholds[-1]])
# plot grid
grid_step = int(plot_max / 10.0)
for i in range(grid_step, plot_max - grid_step + 1, grid_step):
plt.plot([i, i], [0, 100], 'k', alpha=0.1)
for i in range(10, 91, 10):
plt.plot([0, plot_max], [i, i], 'k', alpha=0.1)
plot_title = title
if rotation_threshold >= 0:
plot_title += (
' ' if title else
'') + r'$\alpha_{th}$' + '={} deg'.format(rotation_threshold)
plt.title(plot_title)
full_path = path.join(output_folder, PLOT_FILENAME + '.png')
safe_remove_file(full_path, force)
plt.savefig(full_path, bbox_inches='tight')