def Test(ds_left, ds_right, reproj_left):
model.eval()
if args.cuda:
ds_left, ds_right, reproj_left = ds_left.cuda(), ds_right.cuda(), reproj_left.cuda()
ds_left = ds_left.unsqueeze(0)
ds_right = ds_right.unsqueeze(0)
reproj_left = reproj_left.unsqueeze(0)
with torch.no_grad():
t1 = time.time()
output3 = model(ds_left, ds_right)[0]
b, h, w = output3.shape
depth3 = submodule.reprojection()(output3, reproj_left)
dur = time.time() - t1
print("dur: ", dur)
cp_z = depth3[:, 2, :, :]
min_val = cp_z.min()
max_val = cp_z.max()
cp_z = ((cp_z - min_val) / (max_val - min_val)).data.cpu().numpy()
heat_cp = cv2.applyColorMap(np.uint8(255 * np.moveaxis(cp_z, source=[0, 1, 2], destination=[2, 0, 1])),
cv2.COLORMAP_JET)
depth3 = depth3.squeeze(0).permute([1,2,0]).data.cpu().numpy()
return heat_cp, depth3
def Test(sample):
model.training = False
model.eval()
left = sample['left'] # [B, 3, H, W]
right = sample['right']
depth = sample['depth'] # [B, H, W]
mask = sample['mask']
reproj_left = sample['reprojection']
if args.cuda:
left, right, mask, reproj_left = left.cuda(), right.cuda(), mask.cuda(
), reproj_left.cuda()
# mask = mask > 0.6
b, c, h, w = depth.shape
count = b
for i in range(b):
if depth[i][mask[i]].numel() < depth[i].numel() * 0.1:
mask[i] = mask[i] < 0
count -= 1
if count < 1:
return -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
with torch.no_grad():
output3 = model(left, right)[0]
b, h, w = output3.shape
bg, c, hg, wg = depth.shape
ds = hg / h
output3 = output3.unsqueeze(1)
output3 = F.upsample(output3, [hg, wg], mode='bilinear')
output3 = torch.mul(output3, ds)
output3 = torch.squeeze(output3, 1)
depth3 = submodule.reprojection()(output3, reproj_left)
output3 = output3.data.cpu()
depth3 = depth3.data.cpu()
mask = mask.data.cpu()
gt_z = depth[:, 2, :, :]
cp_z = depth3[:, 2, :, :]
z_mask = mask[:, 2, :, :]
mae_z = torch.mean(torch.abs(gt_z[z_mask] - cp_z[z_mask]))
mae_xyz = torch.mean(torch.abs(depth[mask] - depth3[mask]))
bad2_mask = torch.abs(gt_z - cp_z) > 2
bad1_mask = torch.abs(gt_z - cp_z) > 1
bad05_mask = torch.abs(gt_z - cp_z) > 0.5
bad10_mask = torch.abs(gt_z - cp_z) > 10
bad2 = gt_z[bad2_mask & z_mask].numel() / gt_z[z_mask].numel()
bad1 = gt_z[bad1_mask & z_mask].numel() / gt_z[z_mask].numel()
bad05 = gt_z[bad05_mask & z_mask].numel() / gt_z[z_mask].numel()
bad10 = gt_z[bad10_mask & z_mask].numel() / gt_z[z_mask].numel()
return output3, depth3, mask, mae_xyz.item(), mae_z.item(
), bad10, bad2, bad1, bad05, count
def Test(ds_left, ds_right, rec_left_gt, rec_right_gt, mask_left, mask_right,
reproj_left, reproj_right, bb, ff):
model.eval()
if args.cuda:
ds_left, ds_right, rec_left_gt, rec_right_gt, mask_left, mask_right, reproj_left, reproj_right, bb, ff = ds_left.cuda(
), ds_right.cuda(), rec_left_gt.cuda(), rec_right_gt.cuda(
), mask_left.cuda(), mask_right.cuda(), reproj_left.cuda(
), reproj_right.cuda(), bb.cuda(), ff.cuda()
mask_left = mask_left > 0.6
b, c, h, w = rec_left_gt.shape
count = b
for i in range(b):
if rec_left_gt[i][mask_left[i]].numel() < rec_left_gt[i].numel() * 0.1:
mask_left[i] = mask_left[i] < 0
count -= 1
if count < 1:
return -1, -1, -1, -1, -1, -1, -1
# ds_left = ds_left.unsqueeze(0)
# ds_right = ds_right.unsqueeze(0)
# reproj_left = reproj_left.unsqueeze(0)
with torch.no_grad():
output1, output3 = model(ds_left, ds_right)
b, h, w = output3.shape
b, c, hg, wg = rec_left_gt.shape
ds = hg / h
diff = torch.abs(output1 - output3)
print("diff: ", diff.sum())
output3 = output3.unsqueeze(1)
output3 = F.upsample(output3, [hg, wg], mode='bilinear')
output3 = torch.mul(output3, ds)
output3 = torch.squeeze(output3, 1)
depth3 = submodule.reprojection()(output3, reproj_left)
gt_z = rec_left_gt[:, 2, :, :]
cp_z = depth3[:, 2, :, :]
z_mask = mask_left[:, 2, :, :]
mae_z = torch.mean(torch.abs(gt_z[z_mask] - cp_z[z_mask]))
mae_xyz = torch.mean(torch.abs(rec_left_gt[mask_left] - depth3[mask_left]))
bad2_mask = torch.abs(gt_z - cp_z) > 2
bad1_mask = torch.abs(gt_z - cp_z) > 1
bad05_mask = torch.abs(gt_z - cp_z) > 0.5
bad10_mask = torch.abs(gt_z - cp_z) > 10
bad2 = gt_z[bad2_mask & z_mask].numel() / gt_z[z_mask].numel()
bad1 = gt_z[bad1_mask & z_mask].numel() / gt_z[z_mask].numel()
bad05 = gt_z[bad05_mask & z_mask].numel() / gt_z[z_mask].numel()
bad10 = gt_z[bad10_mask & z_mask].numel() / gt_z[z_mask].numel()
return mae_xyz, mae_z, bad10, bad2, bad1, bad05, count
def train(sample):
model.train()
left = sample['left'] # [B, 3, H, W]
right = sample['right']
depth = sample['depth'] # [B, H, W]
mask = sample['mask']
reproj_left = sample['reprojection']
if args.cuda:
left, right, depth, mask, reproj_left = left.cuda(), right.cuda(
), depth.cuda(), mask.cuda(), reproj_left.cuda()
# mask = mask > 0.6
b, c, h, w = depth.shape
count = b
for i in range(b):
if depth[i][mask[i]].numel() < depth[i].numel() * 0.1:
mask[i] = mask[i] < 0
count -= 1
if count < 1:
return -1, -1, -1, -1, -1
optimizer.zero_grad()
if args.model == 'GwcNet_GC':
output = model(left, right)
output0 = output[0]
output1 = output[1]
output2 = output[2]
output3 = output[3]
b, h, w = output1.shape
bg, c, hg, wg = depth.shape
output0 = output0.unsqueeze(1)
output1 = output1.unsqueeze(1)
output2 = output2.unsqueeze(1)
output3 = output3.unsqueeze(1)
ds = hg / h
output0 = F.upsample(output0, [hg, wg], mode='bilinear')
output1 = F.upsample(output1, [hg, wg], mode='bilinear')
output2 = F.upsample(output2, [hg, wg], mode='bilinear')
output3 = F.upsample(output3, [hg, wg], mode='bilinear')
output0 = torch.mul(output0, ds)
output1 = torch.mul(output1, ds) # 上采样后要乘以采样率
output2 = torch.mul(output2, ds)
output3 = torch.mul(output3, ds)
output0 = torch.squeeze(output0, 1)
output1 = torch.squeeze(output1, 1)
output2 = torch.squeeze(output2, 1)
output3 = torch.squeeze(output3, 1) # 输出是batch*height*width
depth0 = submodule.reprojection()(output0, reproj_left)
depth1 = submodule.reprojection()(output1, reproj_left)
depth2 = submodule.reprojection()(output2, reproj_left)
depth3 = submodule.reprojection()(output3, reproj_left)
# fn = torch.nn.MSELoss()
loss = 0.5 * F.smooth_l1_loss(
depth0[mask], depth[mask]) + 0.5 * F.smooth_l1_loss(
depth1[mask], depth[mask]) + 0.7 * F.smooth_l1_loss(
depth2[mask], depth[mask]) + F.smooth_l1_loss(
depth3[mask], depth[mask])
elif args.model == 'basic':
output3 = model(ds_left, ds_right)
output3 = output3.unsqueeze(1)
b, d, h, w = output3.shape
bg, c, hg, wg = rec_left_gt.shape
ds = hg / h
output3 = F.upsample(output3, [hg, wg], mode='bilinear')
output3 = torch.mul(output3, ds)
output3 = torch.squeeze(output3, 1)
depth3 = submodule.reprojection()(output3, reproj_left)
loss = F.l1_loss(rec_left_gt[mask_left],
depth3[mask_left],
size_average=True)
loss.backward()
optimizer.step()
# display.display_color_disparity_depth(step, writer, ds_left, output3.unsqueeze(1), depth3, is_return_img=False)
return loss.item(), count, output3, depth3, mask
def train(sample):
model.train()
left = sample['left'] # [B, 3, H, W]
right = sample['right']
disp = sample['disp'] # [B, H, W]
mask = sample['mask']
if args.cuda:
left, right, disp, mask = left.cuda(), right.cuda(), disp.cuda(
), mask.cuda()
mask = mask > 0.6
b, c, h, w = disp.shape
count = b
for i in range(b):
if disp[i][mask[i]].numel() < disp[i].numel() * 0.1:
mask[i] = mask[i] < 0
count -= 1
if count < 1:
return -1, -1, -1
optimizer.zero_grad()
if args.use_hourglass == True:
output1, output2, output3 = model(left, right)
b, h, w = output1.shape
bg, c, hg, wg = disp.shape
output1 = output1.unsqueeze(1)
output2 = output2.unsqueeze(1)
output3 = output3.unsqueeze(1)
ds = hg / h
output1 = F.upsample(output1, [hg, wg], mode='bilinear')
output2 = F.upsample(output2, [hg, wg], mode='bilinear')
output3 = F.upsample(output3, [hg, wg], mode='bilinear')
output1 = torch.mul(output1, ds) # 上采样后要乘以采样率
output2 = torch.mul(output2, ds)
output3 = torch.mul(output3, ds)
# output1 = torch.squeeze(output1, 1)
# output2 = torch.squeeze(output2, 1)
# output3 = torch.squeeze(output3, 1) # 输出是batch*height*width
# depth1 = submodule.reprojection()(output1, reproj_left)
# depth2 = submodule.reprojection()(output2, reproj_left)
# depth3 = submodule.reprojection()(output3, reproj_left)
# gt = rec_left_gt[0].cpu().numpy()
# np.save('gt.npy', gt)
# bb = bb[0].cpu().numpy()
# np.save('b.npy', bb)
# ff = ff[0].cpu().numpy()
# np.save('f.npy', ff)
# disp = output3[0].cpu().detach().numpy()
# np.save('disparity.npy', disp)
# z = depth3[0].cpu().detach().numpy()
# np.save('depth.npy', z)
# fn = torch.nn.MSELoss()
# loss = 0.5 * F.smooth_l1_loss(depth1[mask_left], rec_left_gt[mask_left]) + 0.7 * F.smooth_l1_loss(depth2[mask_left], rec_left_gt[mask_left]) + F.smooth_l1_loss(depth3[mask_left], rec_left_gt[mask_left])
loss = 0.5 * F.smooth_l1_loss(
output1[mask], disp[mask]) + 0.7 * F.smooth_l1_loss(
output2[mask], disp[mask]) + F.smooth_l1_loss(
output3[mask], disp[mask])
elif args.model == 'basic':
output3 = model(ds_left, ds_right)
output3 = output3.unsqueeze(1)
b, d, h, w = output3.shape
bg, c, hg, wg = rec_left_gt.shape
ds = hg / h
output3 = F.upsample(output3, [hg, wg], mode='bilinear')
output3 = torch.mul(output3, ds)
output3 = torch.squeeze(output3, 1)
depth3 = submodule.reprojection()(output3, reproj_left)
loss = F.l1_loss(rec_left_gt[mask_left],
depth3[mask_left],
size_average=True)
loss.backward()
optimizer.step()
# display.display_color_disparity_depth(step, writer, ds_left, output3.unsqueeze(1), depth3, is_return_img=False)
return loss.item(), count, output3