def forward(self, batch, return_logs=False, progress=0.0): """ Processes a batch. Parameters ---------- batch : dict Input batch return_logs : bool True if logs are stored progress : Training progress percentage Returns ------- output : dict Dictionary containing a "loss" scalar and different metrics and predictions for logging and downstream usage. """ if not self.training: # If not training, no need for self-supervised loss return SfmModel.forward(self, batch) else: # Calculate predicted depth and pose output output = super().forward(batch, return_logs=return_logs) # Introduce poses ground_truth poses_gt = [[], []] poses_gt[0], poses_gt[1] = torch.zeros((6, 4, 4)), torch.zeros((6, 4, 4)) for i in range(6): poses_gt[0][i] = batch['pose_context'][0][i].inverse() @ batch['pose'][i] poses_gt[1][i] = batch['pose_context'][1][i].inverse() @ batch['pose'][i] poses_gt = [Pose(poses_gt[0]), Pose(poses_gt[1])] multiview_loss = self.multiview_photometric_loss( batch['rgb_original'], batch['rgb_context_original'], output['inv_depths'], poses_gt, batch['intrinsics'], batch['extrinsics'], return_logs=return_logs, progress=progress) # loss = multiview_loss['loss'] loss = 0. # Calculate supervised loss supervision_loss = self.supervised_loss(output['inv_depths'], depth2inv(batch['depth']), return_logs=return_logs, progress=progress) loss += self.supervised_loss_weight * supervision_loss['loss'] # Return loss and metrics return { 'loss': loss, **merge_outputs(merge_outputs(multiview_loss, supervision_loss), output) }
def forward(self, batch, return_logs=False, progress=0.0): """ Processes a batch. Parameters ---------- batch : dict Input batch return_logs : bool True if logs are stored progress : Training progress percentage Returns ------- output : dict Dictionary containing a "loss" scalar and different metrics and predictions for logging and downstream usage. """ # Calculate predicted depth and pose output output = super().forward(batch, return_logs=return_logs) if not self.training: # If not training, no need for self-supervised loss return output else: # Otherwise, calculate self-supervised loss self_sup_output = self.self_supervised_loss( batch['rgb_original'], batch['rgb_context_original'], output['inv_depths'], output['poses'], batch['intrinsics'], return_logs=return_logs, progress=progress) # Return loss and metrics return { 'loss': self_sup_output['loss'], **merge_outputs(output, self_sup_output), }
def forward(self, batch, return_logs=False, progress=0.0): """ Processes a batch. Parameters ---------- batch : dict Input batch return_logs : bool True if logs are stored progress : Training progress percentage Returns ------- output : dict Dictionary containing a "loss" scalar and different metrics and predictions for logging and downstream usage. """ if not self.training: # If not training, no need for self-supervised loss return SfmModel.forward(self, batch) else: if self.supervised_loss_weight == 1.: # If no self-supervision, no need to calculate loss self_sup_output = SfmModel.forward(self, batch) loss = torch.tensor([0.]).type_as(batch['rgb']) else: # Otherwise, calculate and weight self-supervised loss self_sup_output = SelfSupModel_fisheye_valeo.forward(self, batch) loss = (1.0 - self.supervised_loss_weight) * self_sup_output['loss'] reproj_output = self.reprojected_loss( batch['depth'], inv2depth(self_sup_output['inv_depths']), self_sup_output['poses'], batch['path_to_theta_lut'], batch['path_to_ego_mask'], batch['intrinsics_poly_coeffs'], batch['intrinsics_principal_point'], batch['intrinsics_scale_factors'], batch['path_to_theta_lut_context'], batch['path_to_ego_mask_context'], batch['intrinsics_poly_coeffs_context'], batch['intrinsics_principal_point_context'], batch['intrinsics_scale_factors_context'], return_logs=return_logs, progress=progress) # Calculate and weight supervised loss loss += self.reprojected_loss_weight * reproj_output['loss'] # Merge and return outputs return { 'loss': loss, **merge_outputs(self_sup_output, reproj_output), }
def forward(self, batch, mask_ego=True, return_logs=False, progress=0.0): """ Processes a batch. Parameters ---------- batch : dict Input batch return_logs : bool True if logs are stored progress : Training progress percentage Returns ------- output : dict Dictionary containing a "loss" scalar and different metrics and predictions for logging and downstream usage. """ # Calculate predicted depth and pose output output = super().forward(batch, return_logs=return_logs) if not self.training: # If not training, no need for self-supervised loss return output else: with torch.no_grad(): context_inv_depths = [] n_context = len(batch['rgb_context_original']) for i_context in range(n_context): context = {'rgb': batch['rgb_context_original'][i_context]} output_context = super().forward(context, return_logs=return_logs) context_inv_depths.append(output_context['inv_depths']) # Otherwise, calculate self-supervised loss self_sup_output = self.self_supervised_loss( batch['rgb_original'], batch['rgb_context_original'], output['inv_depths'], context_inv_depths, output['poses'], batch['path_to_theta_lut'], batch['path_to_ego_mask'], batch['intrinsics_poly_coeffs'], batch['intrinsics_principal_point'], batch['intrinsics_scale_factors'], batch['path_to_theta_lut_context'], batch['path_to_ego_mask_context'], batch['intrinsics_poly_coeffs_context'], batch['intrinsics_principal_point_context'], batch['intrinsics_scale_factors_context'], batch['same_timestep_as_origin_context'], batch['pose_matrix_context'], return_logs=return_logs, progress=progress) # Return loss and metrics return { 'loss': self_sup_output['loss'], **merge_outputs(output, self_sup_output), }
def forward(self, batch, return_logs=False, progress=0.0, **kwargs): """ Processes a batch. Parameters ---------- batch : dict Input batch return_logs : bool True if logs are stored progress : Training progress percentage Returns ------- output : dict Dictionary containing a "loss" scalar and different metrics and predictions for logging and downstream usage. """ if not self.training: # If not training, no need for self-supervised loss return SfmModel.forward(self, batch, return_logs=return_logs, **kwargs) else: if self.supervised_loss_weight == 1.: # If no self-supervision, no need to calculate loss self_sup_output = SfmModel.forward(self, batch, return_logs=return_logs, **kwargs) loss = torch.tensor([0.]).type_as(batch['rgb']) else: # Otherwise, calculate and weight self-supervised loss self_sup_output = SelfSupModel.forward( self, batch, return_logs=return_logs, progress=progress, **kwargs) loss = (1.0 - self.supervised_loss_weight) * self_sup_output['loss'] # Calculate and weight supervised loss sup_output = self.supervised_loss( self_sup_output['inv_depths'], depth2inv(batch['depth']), return_logs=return_logs, progress=progress) loss += self.supervised_loss_weight * sup_output['loss'] if 'inv_depths_rgbd' in self_sup_output: sup_output2 = self.supervised_loss( self_sup_output['inv_depths_rgbd'], depth2inv(batch['depth']), return_logs=return_logs, progress=progress) loss += self.weight_rgbd * self.supervised_loss_weight * sup_output2['loss'] if 'depth_loss' in self_sup_output: loss += self_sup_output['depth_loss'] # Merge and return outputs return { 'loss': loss, **merge_outputs(self_sup_output, sup_output), }
def forward(self, batch, mask_ego=True, return_logs=False, progress=0.0): """ Processes a batch. Parameters ---------- batch : dict Input batch return_logs : bool True if logs are stored progress : Training progress percentage Returns ------- output : dict Dictionary containing a "loss" scalar and different metrics and predictions for logging and downstream usage. """ # Calculate predicted depth and pose output output = super().forward(batch, return_logs=return_logs) if not self.training: # If not training, no need for self-supervised loss return output else: # Otherwise, calculate self-supervised loss self_sup_output = self.self_supervised_loss( batch['rgb_original'], batch['rgb_temporal_context_original'], batch['rgb_geometric_context_original'], batch['rgb_geometric_context_temporal_context_original'], output['inv_depths'], output['poses_temporal_context'], batch['pose_matrix_geometric_context'], output['poses_geometric_context_temporal_context'], batch['camera_type'], batch['intrinsics_poly_coeffs'], batch['intrinsics_principal_point'], batch['intrinsics_scale_factors'], batch['intrinsics_K'], batch['intrinsics_k'], batch['intrinsics_p'], batch['path_to_ego_mask'], batch['camera_type_geometric_context'], batch['intrinsics_poly_coeffs_geometric_context'], batch['intrinsics_principal_point_geometric_context'], batch['intrinsics_scale_factors_geometric_context'], batch['intrinsics_K_geometric_context'], batch['intrinsics_k_geometric_context'], batch['intrinsics_p_geometric_context'], batch['path_to_ego_mask_geometric_context'], return_logs=return_logs, progress=progress) pose_consistency_loss = self._pose_consistency_loss( output['poses_temporal_context'], output['poses_geometric_context_temporal_context'], batch['camera_type_geometric_context'], batch['pose_matrix'], batch['pose_matrix_geometric_context_absolute']) # Return loss and metrics return { 'loss': self_sup_output['loss'] + pose_consistency_loss['loss'], **merge_outputs(output, self_sup_output), }
def forward(self, batch, return_logs=False, progress=0.0): """ Processes a batch. Parameters ---------- batch : dict Input batch return_logs : bool True if logs are stored progress : Training progress percentage Returns ------- output : dict Dictionary containing a "loss" scalar and different metrics and predictions for logging and downstream usage. """ # Calculate predicted depth and pose output output = super().forward(batch, return_logs=return_logs) if not self.training: # If not training, no need for self-supervised loss return output else: # Otherwise, calculate self-supervised loss self_sup_output = self.self_supervised_loss( batch['rgb_original'], batch['rgb_context_original'], output['inv_depths'], output['poses'], batch['intrinsics'], return_logs=return_logs, progress=progress) if len(batch['rgb_context'] ) == 2 and self.pose_consistency_loss_weight != 0.: pose_contexts = self.compute_poses(batch['rgb_context'][0], [batch['rgb_context'][1]]) pose_consistency_output = self.pose_consistency_loss( invert_pose(output['poses'][0].item()), output['poses'][1].item(), pose_contexts[0].item()) pose_consistency_output[ 'loss'] *= self.pose_consistency_loss_weight output = merge_outputs(output, pose_consistency_output) if self.identity_pose_loss_weight != 0: # Identity pose loss. pose(I_t, I_t+1) = pose(I_t+1, It)^⁻1 pose_vec_minus1_0 = self.pose_net(batch['rgb_context'][0], batch['rgb']) pose_vec_minus1_0 = Pose.from_vec(pose_vec_minus1_0, self.rotation_mode) identity_pose_output_minus1_0 = self.identity_pose_loss( output['poses'][0].item(), invert_pose(pose_vec_minus1_0)) pose_vec_01 = self.pose_net(batch['rgb_context'][1], batch['rgb']) pose_vec_01 = Pose.from_vec(pose_vec_01, self.rotation_mode) identity_pose_output_01 = self.identity_pose_loss( output['poses'][1].item(), invert_pose(pose_vec_01)) identity_pose_output_minus1_0[ 'loss'] *= self.identity_pose_loss_weight identity_pose_output_01[ 'loss'] *= self.identity_pose_loss_weight output = merge_outputs(output, identity_pose_output_minus1_0, identity_pose_output_01) if self.temporal_consistency_loss_weight != 0: # Temporal consistency: D_t = proj_on_t(D_t-1) + pose_t-1_t[3,3] temporal_consistency_loss = [] # for each ref image for j, (ref_image, pose) in enumerate( zip(batch['rgb_context'], output['poses'])): ref_warped_depths = warp_inv_depth(output['inv_depths'], batch['intrinsics'], batch['intrinsics'], pose) # add z from pose to warped depth ref_warped_depths = [ ref_warped_depth + pose[:, 3, 3] for ref_warped_depth in ref_warped_depths ] ref_inv_depths = self.compute_inv_depths(ref_image) ref_depths = [ inv2depth(ref_inv_depths[i]) for i in range(len(ref_inv_depths)) ] l1_loss = self.temporal_consistency_loss( ref_warped_depths, ref_depths) temporal_consistency_loss.append([i for i in l1_loss]) temporal_consistency_loss = temporal_consistency_loss.mean( ) * self.temporal_consistency_loss_weight output = merge_outputs(output, temporal_consistency_loss) # Return loss and metrics return { 'loss': self_sup_output['loss'], **merge_outputs(output, self_sup_output), }