def forward(self, batch, return_logs=False, progress=0.0):
"""
Processes a batch.
Parameters
----------
batch : dict
Input batch
return_logs : bool
True if logs are stored
progress :
Training progress percentage
Returns
-------
output : dict
Dictionary containing a "loss" scalar and different metrics and predictions
for logging and downstream usage.
"""
if not self.training:
# If not training, no need for self-supervised loss
return SfmModel.forward(self, batch)
else:
# Calculate predicted depth and pose output
output = super().forward(batch, return_logs=return_logs)
# Introduce poses ground_truth
poses_gt = [[], []]
poses_gt[0], poses_gt[1] = torch.zeros((6, 4, 4)), torch.zeros((6, 4, 4))
for i in range(6):
poses_gt[0][i] = batch['pose_context'][0][i].inverse() @ batch['pose'][i]
poses_gt[1][i] = batch['pose_context'][1][i].inverse() @ batch['pose'][i]
poses_gt = [Pose(poses_gt[0]), Pose(poses_gt[1])]
multiview_loss = self.multiview_photometric_loss(
batch['rgb_original'], batch['rgb_context_original'],
output['inv_depths'], poses_gt, batch['intrinsics'], batch['extrinsics'],
return_logs=return_logs, progress=progress)
# loss = multiview_loss['loss']
loss = 0.
# Calculate supervised loss
supervision_loss = self.supervised_loss(output['inv_depths'], depth2inv(batch['depth']),
return_logs=return_logs, progress=progress)
loss += self.supervised_loss_weight * supervision_loss['loss']
# Return loss and metrics
return {
'loss': loss,
**merge_outputs(merge_outputs(multiview_loss, supervision_loss), output)
}
def forward(self, batch, return_logs=False, progress=0.0):
"""
Processes a batch.
Parameters
----------
batch : dict
Input batch
return_logs : bool
True if logs are stored
progress :
Training progress percentage
Returns
-------
output : dict
Dictionary containing a "loss" scalar and different metrics and predictions
for logging and downstream usage.
"""
# Calculate predicted depth and pose output
output = super().forward(batch, return_logs=return_logs)
if not self.training:
# If not training, no need for self-supervised loss
return output
else:
# Otherwise, calculate self-supervised loss
self_sup_output = self.self_supervised_loss(
batch['rgb_original'], batch['rgb_context_original'],
output['inv_depths'], output['poses'], batch['intrinsics'],
return_logs=return_logs, progress=progress)
# Return loss and metrics
return {
'loss': self_sup_output['loss'],
**merge_outputs(output, self_sup_output),
}
def forward(self, batch, return_logs=False, progress=0.0):
"""
Processes a batch.
Parameters
----------
batch : dict
Input batch
return_logs : bool
True if logs are stored
progress :
Training progress percentage
Returns
-------
output : dict
Dictionary containing a "loss" scalar and different metrics and predictions
for logging and downstream usage.
"""
if not self.training:
# If not training, no need for self-supervised loss
return SfmModel.forward(self, batch)
else:
if self.supervised_loss_weight == 1.:
# If no self-supervision, no need to calculate loss
self_sup_output = SfmModel.forward(self, batch)
loss = torch.tensor([0.]).type_as(batch['rgb'])
else:
# Otherwise, calculate and weight self-supervised loss
self_sup_output = SelfSupModel_fisheye_valeo.forward(self, batch)
loss = (1.0 - self.supervised_loss_weight) * self_sup_output['loss']
reproj_output = self.reprojected_loss(
batch['depth'], inv2depth(self_sup_output['inv_depths']),
self_sup_output['poses'],
batch['path_to_theta_lut'],
batch['path_to_ego_mask'],
batch['intrinsics_poly_coeffs'],
batch['intrinsics_principal_point'],
batch['intrinsics_scale_factors'],
batch['path_to_theta_lut_context'],
batch['path_to_ego_mask_context'],
batch['intrinsics_poly_coeffs_context'],
batch['intrinsics_principal_point_context'],
batch['intrinsics_scale_factors_context'],
return_logs=return_logs, progress=progress)
# Calculate and weight supervised loss
loss += self.reprojected_loss_weight * reproj_output['loss']
# Merge and return outputs
return {
'loss': loss,
**merge_outputs(self_sup_output, reproj_output),
}
def forward(self, batch, mask_ego=True, return_logs=False, progress=0.0):
"""
Processes a batch.
Parameters
----------
batch : dict
Input batch
return_logs : bool
True if logs are stored
progress :
Training progress percentage
Returns
-------
output : dict
Dictionary containing a "loss" scalar and different metrics and predictions
for logging and downstream usage.
"""
# Calculate predicted depth and pose output
output = super().forward(batch, return_logs=return_logs)
if not self.training:
# If not training, no need for self-supervised loss
return output
else:
with torch.no_grad():
context_inv_depths = []
n_context = len(batch['rgb_context_original'])
for i_context in range(n_context):
context = {'rgb': batch['rgb_context_original'][i_context]}
output_context = super().forward(context, return_logs=return_logs)
context_inv_depths.append(output_context['inv_depths'])
# Otherwise, calculate self-supervised loss
self_sup_output = self.self_supervised_loss(
batch['rgb_original'], batch['rgb_context_original'],
output['inv_depths'], context_inv_depths, output['poses'],
batch['path_to_theta_lut'],
batch['path_to_ego_mask'],
batch['intrinsics_poly_coeffs'],
batch['intrinsics_principal_point'],
batch['intrinsics_scale_factors'],
batch['path_to_theta_lut_context'],
batch['path_to_ego_mask_context'],
batch['intrinsics_poly_coeffs_context'],
batch['intrinsics_principal_point_context'],
batch['intrinsics_scale_factors_context'],
batch['same_timestep_as_origin_context'],
batch['pose_matrix_context'],
return_logs=return_logs, progress=progress)
# Return loss and metrics
return {
'loss': self_sup_output['loss'],
**merge_outputs(output, self_sup_output),
}
def forward(self, batch, return_logs=False, progress=0.0, **kwargs):
"""
Processes a batch.
Parameters
----------
batch : dict
Input batch
return_logs : bool
True if logs are stored
progress :
Training progress percentage
Returns
-------
output : dict
Dictionary containing a "loss" scalar and different metrics and predictions
for logging and downstream usage.
"""
if not self.training:
# If not training, no need for self-supervised loss
return SfmModel.forward(self, batch, return_logs=return_logs, **kwargs)
else:
if self.supervised_loss_weight == 1.:
# If no self-supervision, no need to calculate loss
self_sup_output = SfmModel.forward(self, batch, return_logs=return_logs, **kwargs)
loss = torch.tensor([0.]).type_as(batch['rgb'])
else:
# Otherwise, calculate and weight self-supervised loss
self_sup_output = SelfSupModel.forward(
self, batch, return_logs=return_logs, progress=progress, **kwargs)
loss = (1.0 - self.supervised_loss_weight) * self_sup_output['loss']
# Calculate and weight supervised loss
sup_output = self.supervised_loss(
self_sup_output['inv_depths'], depth2inv(batch['depth']),
return_logs=return_logs, progress=progress)
loss += self.supervised_loss_weight * sup_output['loss']
if 'inv_depths_rgbd' in self_sup_output:
sup_output2 = self.supervised_loss(
self_sup_output['inv_depths_rgbd'], depth2inv(batch['depth']),
return_logs=return_logs, progress=progress)
loss += self.weight_rgbd * self.supervised_loss_weight * sup_output2['loss']
if 'depth_loss' in self_sup_output:
loss += self_sup_output['depth_loss']
# Merge and return outputs
return {
'loss': loss,
**merge_outputs(self_sup_output, sup_output),
}
def forward(self, batch, mask_ego=True, return_logs=False, progress=0.0):
"""
Processes a batch.
Parameters
----------
batch : dict
Input batch
return_logs : bool
True if logs are stored
progress :
Training progress percentage
Returns
-------
output : dict
Dictionary containing a "loss" scalar and different metrics and predictions
for logging and downstream usage.
"""
# Calculate predicted depth and pose output
output = super().forward(batch, return_logs=return_logs)
if not self.training:
# If not training, no need for self-supervised loss
return output
else:
# Otherwise, calculate self-supervised loss
self_sup_output = self.self_supervised_loss(
batch['rgb_original'],
batch['rgb_temporal_context_original'],
batch['rgb_geometric_context_original'],
batch['rgb_geometric_context_temporal_context_original'],
output['inv_depths'],
output['poses_temporal_context'],
batch['pose_matrix_geometric_context'],
output['poses_geometric_context_temporal_context'],
batch['camera_type'],
batch['intrinsics_poly_coeffs'],
batch['intrinsics_principal_point'],
batch['intrinsics_scale_factors'],
batch['intrinsics_K'],
batch['intrinsics_k'],
batch['intrinsics_p'],
batch['path_to_ego_mask'],
batch['camera_type_geometric_context'],
batch['intrinsics_poly_coeffs_geometric_context'],
batch['intrinsics_principal_point_geometric_context'],
batch['intrinsics_scale_factors_geometric_context'],
batch['intrinsics_K_geometric_context'],
batch['intrinsics_k_geometric_context'],
batch['intrinsics_p_geometric_context'],
batch['path_to_ego_mask_geometric_context'],
return_logs=return_logs,
progress=progress)
pose_consistency_loss = self._pose_consistency_loss(
output['poses_temporal_context'],
output['poses_geometric_context_temporal_context'],
batch['camera_type_geometric_context'], batch['pose_matrix'],
batch['pose_matrix_geometric_context_absolute'])
# Return loss and metrics
return {
'loss':
self_sup_output['loss'] + pose_consistency_loss['loss'],
**merge_outputs(output, self_sup_output),
}
def forward(self, batch, return_logs=False, progress=0.0):
"""
Processes a batch.
Parameters
----------
batch : dict
Input batch
return_logs : bool
True if logs are stored
progress :
Training progress percentage
Returns
-------
output : dict
Dictionary containing a "loss" scalar and different metrics and predictions
for logging and downstream usage.
"""
# Calculate predicted depth and pose output
output = super().forward(batch, return_logs=return_logs)
if not self.training:
# If not training, no need for self-supervised loss
return output
else:
# Otherwise, calculate self-supervised loss
self_sup_output = self.self_supervised_loss(
batch['rgb_original'],
batch['rgb_context_original'],
output['inv_depths'],
output['poses'],
batch['intrinsics'],
return_logs=return_logs,
progress=progress)
if len(batch['rgb_context']
) == 2 and self.pose_consistency_loss_weight != 0.:
pose_contexts = self.compute_poses(batch['rgb_context'][0],
[batch['rgb_context'][1]])
pose_consistency_output = self.pose_consistency_loss(
invert_pose(output['poses'][0].item()),
output['poses'][1].item(), pose_contexts[0].item())
pose_consistency_output[
'loss'] *= self.pose_consistency_loss_weight
output = merge_outputs(output, pose_consistency_output)
if self.identity_pose_loss_weight != 0:
# Identity pose loss. pose(I_t, I_t+1) = pose(I_t+1, It)^⁻1
pose_vec_minus1_0 = self.pose_net(batch['rgb_context'][0],
batch['rgb'])
pose_vec_minus1_0 = Pose.from_vec(pose_vec_minus1_0,
self.rotation_mode)
identity_pose_output_minus1_0 = self.identity_pose_loss(
output['poses'][0].item(), invert_pose(pose_vec_minus1_0))
pose_vec_01 = self.pose_net(batch['rgb_context'][1],
batch['rgb'])
pose_vec_01 = Pose.from_vec(pose_vec_01, self.rotation_mode)
identity_pose_output_01 = self.identity_pose_loss(
output['poses'][1].item(), invert_pose(pose_vec_01))
identity_pose_output_minus1_0[
'loss'] *= self.identity_pose_loss_weight
identity_pose_output_01[
'loss'] *= self.identity_pose_loss_weight
output = merge_outputs(output, identity_pose_output_minus1_0,
identity_pose_output_01)
if self.temporal_consistency_loss_weight != 0:
# Temporal consistency: D_t = proj_on_t(D_t-1) + pose_t-1_t[3,3]
temporal_consistency_loss = [] # for each ref image
for j, (ref_image, pose) in enumerate(
zip(batch['rgb_context'], output['poses'])):
ref_warped_depths = warp_inv_depth(output['inv_depths'],
batch['intrinsics'],
batch['intrinsics'],
pose)
# add z from pose to warped depth
ref_warped_depths = [
ref_warped_depth + pose[:, 3, 3]
for ref_warped_depth in ref_warped_depths
]
ref_inv_depths = self.compute_inv_depths(ref_image)
ref_depths = [
inv2depth(ref_inv_depths[i])
for i in range(len(ref_inv_depths))
]
l1_loss = self.temporal_consistency_loss(
ref_warped_depths, ref_depths)
temporal_consistency_loss.append([i for i in l1_loss])
temporal_consistency_loss = temporal_consistency_loss.mean(
) * self.temporal_consistency_loss_weight
output = merge_outputs(output, temporal_consistency_loss)
# Return loss and metrics
return {
'loss': self_sup_output['loss'],
**merge_outputs(output, self_sup_output),
}
