def validate_without_gt(args, val_loader, disp_net, pose_net, epoch, logger, output_writers=[]):
global device
batch_time = AverageMeter()
losses = AverageMeter(i=4, precision=4)
log_outputs = len(output_writers) > 0
# switch to evaluate mode
disp_net.eval()
pose_net.eval()
end = time.time()
logger.valid_bar.update(0)
for i, (tgt_img, ref_imgs, intrinsics, intrinsics_inv) in enumerate(val_loader):
tgt_img = tgt_img.to(device)
ref_imgs = [img.to(device) for img in ref_imgs]
intrinsics = intrinsics.to(device)
intrinsics_inv = intrinsics_inv.to(device)
# compute output
tgt_depth = [1 / disp_net(tgt_img)]
ref_depths = []
for ref_img in ref_imgs:
ref_depth = [1 / disp_net(ref_img)]
ref_depths.append(ref_depth)
if log_outputs and i < len(output_writers):
if epoch == 0:
output_writers[i].add_image('val Input', tensor2array(tgt_img[0]), 0)
output_writers[i].add_image('val Dispnet Output Normalized',
tensor2array(1/tgt_depth[0][0], max_value=None, colormap='magma'),
epoch)
output_writers[i].add_image('val Depth Output',
tensor2array(tgt_depth[0][0], max_value=10),
epoch)
poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs)
loss_1, loss_3 = compute_photo_and_geometry_loss(tgt_img, ref_imgs, intrinsics, tgt_depth, ref_depths,
poses, poses_inv, args.num_scales, args.with_ssim,
args.with_mask, False, args.padding_mode)
loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs)
loss_1 = loss_1.item()
loss_2 = loss_2.item()
loss_3 = loss_3.item()
loss = loss_1
losses.update([loss, loss_1, loss_2, loss_3])
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
logger.valid_bar.update(i+1)
if i % args.print_freq == 0:
logger.valid_writer.write('valid: Time {} Loss {}'.format(batch_time, losses))
logger.valid_bar.update(len(val_loader))
return losses.avg, ['Total loss', 'Photo loss', 'Smooth loss', 'Consistency loss']
def validate_without_gt(args, val_loader, disp_net, pose_net, epoch, logger):
global device
batch_time = AverageMeter()
losses = AverageMeter(i=4, precision=4)
w1, w2, w3 = args.photo_loss_weight, args.smooth_loss_weight, args.geometry_consistency_weight
poses = np.zeros(
((len(val_loader) - 1) * args.batch_size * (args.sequence_length - 1),
6))
disp_values = np.zeros(((len(val_loader) - 1) * args.batch_size * 3))
# switch to evaluate mode
disp_net.eval()
pose_net.eval()
end = time.time()
logger.valid_bar.update(0)
for i, (tgt_img, ref_imgs, intrinsics,
intrinsics_inv) in enumerate(val_loader):
tgt_img = tgt_img.to(device)
ref_imgs = [img.to(device) for img in ref_imgs]
intrinsics = intrinsics.to(device)
intrinsics_inv = intrinsics_inv.to(device)
# compute output
tgt_depth = [1 / disp_net(tgt_img)]
ref_depths = []
for ref_img in ref_imgs:
ref_depth = [1 / disp_net(ref_img)]
ref_depths.append(ref_depth)
poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs)
loss_1, loss_3 = compute_photo_and_geometry_loss(
tgt_img, ref_imgs, intrinsics, tgt_depth, ref_depths, poses,
poses_inv, args)
loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs)
loss_1 = loss_1.item()
loss_2 = loss_2.item()
loss_3 = loss_3.item()
loss = w1 * loss_1 + w2 * loss_2 + w3 * loss_3
losses.update([loss, loss_1, loss_2, loss_3])
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
logger.valid_bar.update(i + 1)
if i % args.print_freq == 0:
logger.valid_writer.write('valid: Time {} Loss {}'.format(
batch_time, losses))
logger.valid_bar.update(len(val_loader))
return losses.avg, [
'Total loss', 'Photo loss', 'Smooth loss', 'Consistency loss'
]
def train(args, train_loader, disp_net, pose_net, optimizer, epoch_size,
logger, train_writer):
global n_iter, device
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter(precision=4)
w1, w2, w3 = args.photo_loss_weight, args.smooth_loss_weight, args.geometry_consistency_weight
# switch to train mode
disp_net.train()
pose_net.train()
end = time.time()
logger.train_bar.update(0)
for i, (tgt_img, ref_imgs, intrinsics,
intrinsics_inv) in enumerate(train_loader):
log_losses = i > 0 and n_iter % args.print_freq == 0
# measure data loading time
data_time.update(time.time() - end)
tgt_img = tgt_img.to(device)
ref_imgs = [img.to(device) for img in ref_imgs]
intrinsics = intrinsics.to(device)
# compute output
tgt_depth, ref_depths = compute_depth(disp_net, tgt_img, ref_imgs)
poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs,
intrinsics)
#if poses is None:
loss_1, loss_3 = compute_photo_and_geometry_loss(
tgt_img, ref_imgs, intrinsics, tgt_depth, ref_depths, poses,
poses_inv, args.num_scales, args.with_ssim, args.with_mask,
args.with_auto_mask, args.padding_mode)
loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs)
loss = w1 * loss_1 + w2 * loss_2 + w3 * loss_3
if log_losses:
train_writer.add_scalar('photometric_error', loss_1.item(), n_iter)
train_writer.add_scalar('disparity_smoothness_loss', loss_2.item(),
n_iter)
train_writer.add_scalar('geometry_consistency_loss', loss_3.item(),
n_iter)
train_writer.add_scalar('total_loss', loss.item(), n_iter)
# record loss and EPE
losses.update(loss.item(), args.batch_size)
# compute gradient and do Adam step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
with open(args.save_path / args.log_full, 'a') as csvfile:
writer = csv.writer(csvfile, delimiter='\t')
writer.writerow(
[loss.item(),
loss_1.item(),
loss_2.item(),
loss_3.item()])
logger.train_bar.update(i + 1)
if i % args.print_freq == 0:
logger.train_writer.write('Train: Time {} Data {} Loss {}'.format(
batch_time, data_time, losses))
if i >= epoch_size - 1:
break
n_iter += 1
return losses.avg[0]
def validate_without_gt(args, val_loader, disp_net, pose_net, epoch, logger):
global device
batch_time = AverageMeter()
losses = AverageMeter(i=4, precision=4)
w1, w2, w3 = args.photo_loss_weight, args.smooth_loss_weight, args.geometry_consistency_weight
poses = np.zeros(
((len(val_loader) - 1) * args.batch_size * (args.sequence_length - 1),
6))
disp_values = np.zeros(((len(val_loader) - 1) * args.batch_size * 3))
# switch to evaluate mode
disp_net.eval()
pose_net.eval()
end = time.time()
logger.valid_bar.update(0)
all_speeds = torch.tensor([], device=device)
all_p_speeds = torch.tensor([], device=device)
for i, (tgt_img, ref_imgs, intrinsics, intrinsics_inv,
speed) in enumerate(val_loader):
tgt_img = tgt_img.to(device)
ref_imgs = [img.to(device) for img in ref_imgs]
intrinsics = intrinsics.to(device)
intrinsics_inv = intrinsics_inv.to(device)
speed = speed.to(device)
# compute output
tgt_depth = [1 / disp_net(tgt_img)]
ref_depths = []
for ref_img in ref_imgs:
ref_depth = [1 / disp_net(ref_img)]
ref_depths.append(ref_depth)
poses, poses_inv = compute_pose_with_inv(pose_net, tgt_img, ref_imgs)
translations = [x[:, :3] for x in poses]
translations_inv = [x[:, :3] for x in poses_inv]
p_speeds = torch.norm(translations[0], dim=1)
all_speeds = torch.cat((all_speeds, speed))
all_p_speeds = torch.cat((all_p_speeds, p_speeds))
loss_1, loss_3 = compute_photo_and_geometry_loss(tgt_img,
ref_imgs,
intrinsics,
tgt_depth,
ref_depths,
args.with_mask,
poses,
poses_inv,
args.num_scales,
args.with_ssim,
rotation_mode='euler',
padding_mode='zeros')
loss_2 = compute_smooth_loss(tgt_depth, tgt_img, ref_depths, ref_imgs,
args.num_scales)
loss_1 = loss_1.item()
loss_2 = loss_2.item()
loss_3 = loss_3.item()
loss = w1 * loss_1 + w2 * loss_2 + w3 * loss_3
losses.update([loss, loss_1, loss_2, loss_3])
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
logger.valid_bar.update(i + 1)
if i % args.print_freq == 0:
logger.valid_writer.write('valid: Time {} Loss {}'.format(
batch_time, losses))
correlation = pearsonr(all_speeds, all_p_speeds)
logger.valid_bar.update(len(val_loader))
return losses.avg, correlation, [
'Total loss', 'Photo loss', 'Smooth loss', 'Consistency loss'
]
