def validate_without_gt(args, val_loader, disp_net, pose_net, epoch, logger, output_writers=[]): global device batch_time = AverageMeter() losses = AverageMeter(i=4, precision=4) log_outputs = len(output_writers) > 0 # switch to evaluate mode disp_net.eval() pose_net.eval() end = time.time() logger.valid_bar.update(0) for i, (tgt_img, ref_imgs, intrinsics, intrinsics_inv) in enumerate(val_loader): tgt_img = tgt_img.to(device) ref_imgs = [img.to(device) for img in ref_imgs] intrinsics = intrinsics.to(device) intrinsics_inv = intrinsics_inv.to(device) # compute output tgt_depth = [1 / disp_net(tgt_img)] ref_depths = [] for ref_img in ref_imgs: ref_depth = [1 / disp_net(ref_img)] ref_depths.append(ref_depth) if log_outputs and i < len(output_writers): if epoch == 0: output_writers[i].add_image('val Input', tensor2array(tgt_img[0]), 0) output_writers[i].add_image('val Dispnet Output Normalized', tensor2array(1/tgt_depth[0][0], max_value=None, colormap='magma'), epoch) output_writers[i].add_image('val Depth Output', tensor2array(tgt_depth[0][0], max_value=10), epoch) poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs) loss_1, loss_3 = compute_photo_and_geometry_loss(tgt_img, ref_imgs, intrinsics, tgt_depth, ref_depths, poses, poses_inv, args.num_scales, args.with_ssim, args.with_mask, False, args.padding_mode) loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs) loss_1 = loss_1.item() loss_2 = loss_2.item() loss_3 = loss_3.item() loss = loss_1 losses.update([loss, loss_1, loss_2, loss_3]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() logger.valid_bar.update(i+1) if i % args.print_freq == 0: logger.valid_writer.write('valid: Time {} Loss {}'.format(batch_time, losses)) logger.valid_bar.update(len(val_loader)) return losses.avg, ['Total loss', 'Photo loss', 'Smooth loss', 'Consistency loss']
def validate_without_gt(args, val_loader, disp_net, pose_net, epoch, logger): global device batch_time = AverageMeter() losses = AverageMeter(i=4, precision=4) w1, w2, w3 = args.photo_loss_weight, args.smooth_loss_weight, args.geometry_consistency_weight poses = np.zeros( ((len(val_loader) - 1) * args.batch_size * (args.sequence_length - 1), 6)) disp_values = np.zeros(((len(val_loader) - 1) * args.batch_size * 3)) # switch to evaluate mode disp_net.eval() pose_net.eval() end = time.time() logger.valid_bar.update(0) for i, (tgt_img, ref_imgs, intrinsics, intrinsics_inv) in enumerate(val_loader): tgt_img = tgt_img.to(device) ref_imgs = [img.to(device) for img in ref_imgs] intrinsics = intrinsics.to(device) intrinsics_inv = intrinsics_inv.to(device) # compute output tgt_depth = [1 / disp_net(tgt_img)] ref_depths = [] for ref_img in ref_imgs: ref_depth = [1 / disp_net(ref_img)] ref_depths.append(ref_depth) poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs) loss_1, loss_3 = compute_photo_and_geometry_loss( tgt_img, ref_imgs, intrinsics, tgt_depth, ref_depths, poses, poses_inv, args) loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs) loss_1 = loss_1.item() loss_2 = loss_2.item() loss_3 = loss_3.item() loss = w1 * loss_1 + w2 * loss_2 + w3 * loss_3 losses.update([loss, loss_1, loss_2, loss_3]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() logger.valid_bar.update(i + 1) if i % args.print_freq == 0: logger.valid_writer.write('valid: Time {} Loss {}'.format( batch_time, losses)) logger.valid_bar.update(len(val_loader)) return losses.avg, [ 'Total loss', 'Photo loss', 'Smooth loss', 'Consistency loss' ]
def train(args, train_loader, disp_net, pose_net, optimizer, epoch_size, logger, train_writer): global n_iter, device batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter(precision=4) w1, w2, w3 = args.photo_loss_weight, args.smooth_loss_weight, args.geometry_consistency_weight # switch to train mode disp_net.train() pose_net.train() end = time.time() logger.train_bar.update(0) for i, (tgt_img, ref_imgs, intrinsics, intrinsics_inv) in enumerate(train_loader): log_losses = i > 0 and n_iter % args.print_freq == 0 # measure data loading time data_time.update(time.time() - end) tgt_img = tgt_img.to(device) ref_imgs = [img.to(device) for img in ref_imgs] intrinsics = intrinsics.to(device) # compute output tgt_depth, ref_depths = compute_depth(disp_net, tgt_img, ref_imgs) poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs, intrinsics) #if poses is None: loss_1, loss_3 = compute_photo_and_geometry_loss( tgt_img, ref_imgs, intrinsics, tgt_depth, ref_depths, poses, poses_inv, args.num_scales, args.with_ssim, args.with_mask, args.with_auto_mask, args.padding_mode) loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs) loss = w1 * loss_1 + w2 * loss_2 + w3 * loss_3 if log_losses: train_writer.add_scalar('photometric_error', loss_1.item(), n_iter) train_writer.add_scalar('disparity_smoothness_loss', loss_2.item(), n_iter) train_writer.add_scalar('geometry_consistency_loss', loss_3.item(), n_iter) train_writer.add_scalar('total_loss', loss.item(), n_iter) # record loss and EPE losses.update(loss.item(), args.batch_size) # compute gradient and do Adam step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() with open(args.save_path / args.log_full, 'a') as csvfile: writer = csv.writer(csvfile, delimiter='\t') writer.writerow( [loss.item(), loss_1.item(), loss_2.item(), loss_3.item()]) logger.train_bar.update(i + 1) if i % args.print_freq == 0: logger.train_writer.write('Train: Time {} Data {} Loss {}'.format( batch_time, data_time, losses)) if i >= epoch_size - 1: break n_iter += 1 return losses.avg[0]
def validate_without_gt(args, val_loader, disp_net, pose_net, epoch, logger): global device batch_time = AverageMeter() losses = AverageMeter(i=4, precision=4) w1, w2, w3 = args.photo_loss_weight, args.smooth_loss_weight, args.geometry_consistency_weight poses = np.zeros( ((len(val_loader) - 1) * args.batch_size * (args.sequence_length - 1), 6)) disp_values = np.zeros(((len(val_loader) - 1) * args.batch_size * 3)) # switch to evaluate mode disp_net.eval() pose_net.eval() end = time.time() logger.valid_bar.update(0) all_speeds = torch.tensor([], device=device) all_p_speeds = torch.tensor([], device=device) for i, (tgt_img, ref_imgs, intrinsics, intrinsics_inv, speed) in enumerate(val_loader): tgt_img = tgt_img.to(device) ref_imgs = [img.to(device) for img in ref_imgs] intrinsics = intrinsics.to(device) intrinsics_inv = intrinsics_inv.to(device) speed = speed.to(device) # compute output tgt_depth = [1 / disp_net(tgt_img)] ref_depths = [] for ref_img in ref_imgs: ref_depth = [1 / disp_net(ref_img)] ref_depths.append(ref_depth) poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs) translations = [x[:, :3] for x in poses] translations_inv = [x[:, :3] for x in poses_inv] p_speeds = torch.norm(translations[0], dim=1) all_speeds = torch.cat((all_speeds, speed)) all_p_speeds = torch.cat((all_p_speeds, p_speeds)) loss_1, loss_3 = compute_photo_and_geometry_loss(tgt_img, ref_imgs, intrinsics, tgt_depth, ref_depths, args.with_mask, poses, poses_inv, args.num_scales, args.with_ssim, rotation_mode='euler', padding_mode='zeros') loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs, args.num_scales) loss_1 = loss_1.item() loss_2 = loss_2.item() loss_3 = loss_3.item() loss = w1 * loss_1 + w2 * loss_2 + w3 * loss_3 losses.update([loss, loss_1, loss_2, loss_3]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() logger.valid_bar.update(i + 1) if i % args.print_freq == 0: logger.valid_writer.write('valid: Time {} Loss {}'.format( batch_time, losses)) correlation = pearsonr(all_speeds, all_p_speeds) logger.valid_bar.update(len(val_loader)) return losses.avg, correlation, [ 'Total loss', 'Photo loss', 'Smooth loss', 'Consistency loss' ]