def _log_compounded_metrics(self, ATEs, REs, drift_errors):
metric_logger = MetricLogger()
compound_drift_errors = CompoundTranslationRotationDrift(
self.model_name, drift_errors)
metric_logger.log(compound_drift_errors)
CometLogger.get_experiment().log_metric(
"avg_translation_error_percent",
compound_drift_errors.metrics["avg_translation_error_percent"])
CometLogger.get_experiment().log_metric(
"avg_rotation_error_degrees_per_meter", compound_drift_errors.
metrics["avg_rotation_error_degrees_per_meter"])
compound_ATE = CompoundAbsoluteTrajectoryError(self.model_name, ATEs)
metric_logger.log(compound_ATE)
CometLogger.get_experiment().log_metric(
"ATE_trans_RMSE", compound_ATE.metrics["absolute_trajectory_error"]
['ATE_trans_stats']["rmse"])
CometLogger.get_experiment().log_metric(
"ATE_rot_degrees_RMSE",
compound_ATE.metrics["absolute_trajectory_error"]['ATE_rot_stats']
["rmse"])
compound_RE = CompoundRelativeError(self.model_name, REs)
metric_logger.log(compound_RE)
def _log_matrix_poses(self, poses, poses_gt, dataset_name: str,
trajectory_name: str):
"""
Logs the pose in text format where the angle is a rotation matrix:
T00 T01 T02 T03
T10 T11 T12 T13
T20 T21 T22 T23
0 0 0 1
T00 T01 T02 T03 T10 T11 T12 T13 T20 T21 T22 T23
"""
pose_output = ""
pose_gt_output = ""
matrices = Geometry.poses_to_transformations_matrix(
poses[:, 3:], poses[:, :3])
matrices_gt = Geometry.poses_to_transformations_matrix(
poses_gt[:, 3:], poses_gt[:, :3])
for i, _ in enumerate(matrices):
pose_matrix = matrices[i]
pose_matrix_gt = matrices_gt[i]
pose_output = pose_output + f"{pose_matrix[0][0]} {pose_matrix[0][1]} {pose_matrix[0][2]} {pose_matrix[0][3]} " \
f"{pose_matrix[1][0]} {pose_matrix[1][1]} {pose_matrix[1][2]} {pose_matrix[1][3]} " \
f"{pose_matrix[2][0]} {pose_matrix[2][1]} {pose_matrix[2][2]} {pose_matrix[2][3]}"
pose_gt_output = pose_gt_output + f"{pose_matrix_gt[0][0]} {pose_matrix_gt[0][1]} {pose_matrix_gt[0][2]} {pose_matrix_gt[0][3]} " \
f"{pose_matrix_gt[1][0]} {pose_matrix_gt[1][1]} {pose_matrix_gt[1][2]} {pose_matrix_gt[1][3]} " \
f"{pose_matrix_gt[2][0]} {pose_matrix_gt[2][1]} {pose_matrix_gt[2][2]} {pose_matrix_gt[2][3]}"
if i < len(poses) - 1:
pose_output = pose_output + "\n"
pose_gt_output = pose_gt_output + "\n"
metadata = dict()
metadata["title"] = "pose_output_matrix"
metadata["dataset"] = dataset_name
metadata["trajectory"] = trajectory_name
metadata["model"] = self.model_name
filename = f'{metadata["title"]}_{dataset_name}_{trajectory_name}_{metadata["model"]}.txt'
CometLogger.get_experiment().log_asset_data(pose_output,
name=filename,
metadata=metadata)
metadata["title"] = "pose_gt_output_matrix"
filename = f'{metadata["title"]}_{dataset_name}_{trajectory_name}_{metadata["model"]}.txt'
CometLogger.get_experiment().log_asset_data(pose_gt_output,
name=filename,
metadata=metadata)
def load_optimizer(param: Parameters, model: nn.Module) -> Optimizer:
CometLogger.get_experiment().log_parameter("Optimizer", param.optimizer)
CometLogger.get_experiment().log_parameter("Learning rate",
param.learning_rate)
if param.optimizer is "Adagrad":
CometLogger.print("Using Adagrad")
return optim.Adagrad(model.parameters(), lr=param.learning_rate)
elif param.optimizer is "Adam":
CometLogger.print("Using Adam Optimizer")
return optim.Adam(model.parameters(), lr=param.learning_rate)
elif param.optimizer is "RMSProp":
CometLogger.print("Using RMSProp Optimizer")
return optim.RMSprop(model.parameters(), lr=param.learning_rate)
else:
CometLogger.print("Optimizer {} was not implemented".format(
param.optimizer))
raise NotImplementedError()
def _log_quaternion_poses(self, poses, poses_gt, dataset_name: str,
trajectory_name: str):
"""
Logs the pose in text format where the angle is a quaternion:
timestamp tx ty tz qx qy qz qw
"""
pose_output = ""
pose_gt_output = ""
for i, pose in enumerate(poses):
# att.elements[[1, 2, 3, 0]] reorganizes quaternion elements
# from scalar first w-x-y-z to scalar last x-y-z-w
rotation_quat = Geometry.tait_bryan_rotation_to_quaternion(
pose[:3]).elements[[1, 2, 3, 0]]
rotation_quat_gt = Geometry.tait_bryan_rotation_to_quaternion(
poses_gt[i][:3]).elements[[1, 2, 3, 0]]
pose_output = pose_output + f"{i} {pose[3]} {pose[4]} {pose[5]} " \
f"{rotation_quat[0]} {rotation_quat[1]} {rotation_quat[2]} {rotation_quat[3]}"
pose_gt_output = pose_gt_output + f"{i} {poses_gt[i][3]} {poses_gt[i][4]} {poses_gt[i][5]} " \
f"{rotation_quat_gt[0]} {rotation_quat_gt[1]} {rotation_quat_gt[2]} " \
f"{rotation_quat_gt[3]}"
if i < len(poses) - 1:
pose_output = pose_output + "\n"
pose_gt_output = pose_gt_output + "\n"
metadata = dict()
metadata["title"] = "pose_output_quaternion"
metadata["dataset"] = dataset_name
metadata["trajectory"] = trajectory_name
metadata["model"] = self.model_name
filename = f'{metadata["title"]}_{dataset_name}_{trajectory_name}_{metadata["model"]}.txt'
CometLogger.get_experiment().log_asset_data(pose_output,
name=filename,
metadata=metadata)
metadata["title"] = "pose_gt_output_quaternion"
filename = f'{metadata["title"]}_{dataset_name}_{trajectory_name}_{metadata["model"]}.txt'
CometLogger.get_experiment().log_asset_data(pose_gt_output,
name=filename,
metadata=metadata)
def run(self, epochs_number: int) -> nn.Module:
for epoch in self._epochs(epochs_number):
CometLogger.print("=========== Epoch {} ===========".format(epoch))
t0 = time.time()
custom_train_loss, train_benchmark_loss = self._train()
custom_valid_loss, valid_benchmark_loss = self._validate()
t1 = time.time()
epoch_run_time = t1 - t0
self._log_epoch(custom_train_loss, custom_valid_loss, epoch,
epoch_run_time, train_benchmark_loss,
valid_benchmark_loss)
self.early_stopper(custom_valid_loss, self.model, self.optimizer)
if self.early_stopper.early_stop:
CometLogger.get_experiment().log_metric(
"Early stop epoch", epoch + 1)
CometLogger.print("Early stopping")
break
CometLogger.print(
"Training complete, loading the last early stopping checkpoint to memory..."
)
self.model.load_state_dict(self.early_stopper.load_model_checkpoint())
return self.model
def run(self):
trajectory_rotation_losses = []
trajectory_translation_losses = []
drift_errors = []
ATEs = []
REs = []
for dataset_name, trajectory_name, dataloader in self.trajectory_dataloaders:
dataset: AbstractSegmentDataset = dataloader.dataset
print("testing {}, {}".format(trajectory_name, dataset_name))
start = time.time()
predictions, rotation_losses, translation_losses, absolute_ground_truth = self._test(
dataloader)
end = time.time()
last_included_index = self._trim_trajectories(
absolute_ground_truth[:, 3:])
predictions = predictions[:last_included_index + 1]
absolute_ground_truth = absolute_ground_truth[:
last_included_index +
1]
CometLogger.print(
f"Inferred {len(predictions)} poses in {end-start} seconds.\n"
f"Dataset fps: {dataset.framerate}, inference fps {len(predictions)/(end-start)}."
)
trajectory_rotation_losses.append(
(dataset_name, trajectory_name, rotation_losses))
trajectory_translation_losses.append(
(dataset_name, trajectory_name, translation_losses))
plotter = TrajectoryPlotter(trajectory_name, dataset_name,
self.model_name, absolute_ground_truth,
predictions)
CometLogger.get_experiment().log_figure(
figure=plotter.rotation_figure,
figure_name='rotation {} {}'.format(trajectory_name,
dataset_name))
CometLogger.get_experiment().log_figure(
figure=plotter.position_figure,
figure_name='translation {} {}'.format(trajectory_name,
dataset_name))
drift, ATE, RE = self._log_metrics(absolute_ground_truth, dataset,
dataset_name, predictions,
trajectory_name)
drift_errors.append(drift)
ATEs.append(ATE)
REs.append(RE)
self._log_matrix_poses(predictions, absolute_ground_truth,
dataset_name, trajectory_name)
self._log_quaternion_poses(predictions, absolute_ground_truth,
dataset_name, trajectory_name)
self._log_compounded_metrics(ATEs, REs, drift_errors)
losses_figure = self._plot_trajectory_losses(
trajectory_rotation_losses, trajectory_translation_losses)
CometLogger.get_experiment().log_figure(
figure=losses_figure, figure_name="trajectory_losses")
# compute total avg losses
translation_loss = self._complute_total_avg_loss(
trajectory_translation_losses)
rotation_loss = self._complute_total_avg_loss(
trajectory_rotation_losses)
CometLogger.get_experiment().log_metric(
"Total Avg Translation loss (test phase)", translation_loss)
CometLogger.get_experiment().log_metric(
"Total Avg Rotation loss (test phase)", rotation_loss)
def _log_epoch(self, custom_train_loss, custom_valid_loss, epoch,
epoch_run_time, train_benchmark_loss, valid_benchmark_loss):
CometLogger.print("Epoch run time: {}".format(epoch_run_time))
CometLogger.get_experiment().log_metric("epoch run time",
epoch_run_time,
epoch=epoch)
CometLogger.get_experiment().log_metric("mean training loss",
train_benchmark_loss,
epoch=epoch)
CometLogger.get_experiment().log_metric("mean validation loss",
valid_benchmark_loss,
epoch=epoch)
CometLogger.get_experiment().log_metric("custom mean training loss",
custom_train_loss,
epoch=epoch)
CometLogger.get_experiment().log_metric("custom mean validation loss",
custom_valid_loss,
epoch=epoch)
CometLogger.print("Mean train loss: {}, Valid train loss: {}".format(
custom_train_loss, custom_valid_loss))
CometLogger.get_experiment().log_metric("epoch", epoch)
CometLogger.get_experiment().log_epoch_end(epoch_cnt=epoch)
def _train(self) -> tuple:
timer_start_time = time.time()
self.model.train()
losses_sum = 0
benchmark_losses_sum = 0
for i, (input, target) in enumerate(self.train_dataloader):
CometLogger.get_experiment().log_metric("Current batch", i + 1)
CometLogger.get_experiment().log_metric("Total nbr of batches",
len(self.train_dataloader))
# Only log this if we are NOT in a multiprocessing session
if CometLogger.gpu_id is None:
print("--> processing batch {}/{} of size {}".format(
i + 1, len(self.train_dataloader), len(input)))
if cuda_is_available():
with ThreadingTimeout(14400.0) as timeout_ctx1:
input = input.cuda(
non_blocking=self.train_dataloader.pin_memory)
target = target.cuda(
non_blocking=self.train_dataloader.pin_memory)
if not bool(timeout_ctx1):
CometLogger.fatalprint(
'Encountered fatally long delay when moving tensors to GPUs'
)
prediction = self.model.forward(input)
with ThreadingTimeout(14400.0) as timeout_ctx3:
if type(prediction) is tuple:
benchmark_loss = self.benchmark_MSE_loss.compute(
prediction[0], target)
else:
benchmark_loss = self.benchmark_MSE_loss.compute(
prediction, target)
if not bool(timeout_ctx3):
CometLogger.fatalprint(
'Encountered fatally long delay during computation of benchmark loss'
)
with ThreadingTimeout(14400.0) as timeout_ctx4:
benchmark_losses_sum += float(
benchmark_loss.data.cpu().numpy())
if not bool(timeout_ctx4):
CometLogger.fatalprint(
'Encountered fatally long delay during summation of benchmark losses'
)
with ThreadingTimeout(14400.0) as timeout_ctx4:
loss = self.custom_loss.compute(prediction, target)
if not bool(timeout_ctx4):
CometLogger.fatalprint(
'Encountered fatally long delay during computation of the custom loss'
)
self._backpropagate(loss)
with ThreadingTimeout(14400.0) as timeout_ctx6:
losses_sum += float(loss.data.cpu().numpy())
if not bool(timeout_ctx6):
CometLogger.fatalprint(
'Encountered fatally long delay during loss addition')
timer_end_time = time.time()
CometLogger.get_experiment().log_metric(
"Epoch training time", timer_end_time - timer_start_time)
return losses_sum / len(
self.train_dataloader), benchmark_losses_sum / len(
self.train_dataloader)
