def Test(ds_left, ds_right, reproj_left): model.eval() if args.cuda: ds_left, ds_right, reproj_left = ds_left.cuda(), ds_right.cuda(), reproj_left.cuda() ds_left = ds_left.unsqueeze(0) ds_right = ds_right.unsqueeze(0) reproj_left = reproj_left.unsqueeze(0) with torch.no_grad(): t1 = time.time() output3 = model(ds_left, ds_right)[0] b, h, w = output3.shape depth3 = submodule.reprojection()(output3, reproj_left) dur = time.time() - t1 print("dur: ", dur) cp_z = depth3[:, 2, :, :] min_val = cp_z.min() max_val = cp_z.max() cp_z = ((cp_z - min_val) / (max_val - min_val)).data.cpu().numpy() heat_cp = cv2.applyColorMap(np.uint8(255 * np.moveaxis(cp_z, source=[0, 1, 2], destination=[2, 0, 1])), cv2.COLORMAP_JET) depth3 = depth3.squeeze(0).permute([1,2,0]).data.cpu().numpy() return heat_cp, depth3
def Test(sample): model.training = False model.eval() left = sample['left'] # [B, 3, H, W] right = sample['right'] depth = sample['depth'] # [B, H, W] mask = sample['mask'] reproj_left = sample['reprojection'] if args.cuda: left, right, mask, reproj_left = left.cuda(), right.cuda(), mask.cuda( ), reproj_left.cuda() # mask = mask > 0.6 b, c, h, w = depth.shape count = b for i in range(b): if depth[i][mask[i]].numel() < depth[i].numel() * 0.1: mask[i] = mask[i] < 0 count -= 1 if count < 1: return -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 with torch.no_grad(): output3 = model(left, right)[0] b, h, w = output3.shape bg, c, hg, wg = depth.shape ds = hg / h output3 = output3.unsqueeze(1) output3 = F.upsample(output3, [hg, wg], mode='bilinear') output3 = torch.mul(output3, ds) output3 = torch.squeeze(output3, 1) depth3 = submodule.reprojection()(output3, reproj_left) output3 = output3.data.cpu() depth3 = depth3.data.cpu() mask = mask.data.cpu() gt_z = depth[:, 2, :, :] cp_z = depth3[:, 2, :, :] z_mask = mask[:, 2, :, :] mae_z = torch.mean(torch.abs(gt_z[z_mask] - cp_z[z_mask])) mae_xyz = torch.mean(torch.abs(depth[mask] - depth3[mask])) bad2_mask = torch.abs(gt_z - cp_z) > 2 bad1_mask = torch.abs(gt_z - cp_z) > 1 bad05_mask = torch.abs(gt_z - cp_z) > 0.5 bad10_mask = torch.abs(gt_z - cp_z) > 10 bad2 = gt_z[bad2_mask & z_mask].numel() / gt_z[z_mask].numel() bad1 = gt_z[bad1_mask & z_mask].numel() / gt_z[z_mask].numel() bad05 = gt_z[bad05_mask & z_mask].numel() / gt_z[z_mask].numel() bad10 = gt_z[bad10_mask & z_mask].numel() / gt_z[z_mask].numel() return output3, depth3, mask, mae_xyz.item(), mae_z.item( ), bad10, bad2, bad1, bad05, count
def Test(ds_left, ds_right, rec_left_gt, rec_right_gt, mask_left, mask_right, reproj_left, reproj_right, bb, ff): model.eval() if args.cuda: ds_left, ds_right, rec_left_gt, rec_right_gt, mask_left, mask_right, reproj_left, reproj_right, bb, ff = ds_left.cuda( ), ds_right.cuda(), rec_left_gt.cuda(), rec_right_gt.cuda( ), mask_left.cuda(), mask_right.cuda(), reproj_left.cuda( ), reproj_right.cuda(), bb.cuda(), ff.cuda() mask_left = mask_left > 0.6 b, c, h, w = rec_left_gt.shape count = b for i in range(b): if rec_left_gt[i][mask_left[i]].numel() < rec_left_gt[i].numel() * 0.1: mask_left[i] = mask_left[i] < 0 count -= 1 if count < 1: return -1, -1, -1, -1, -1, -1, -1 # ds_left = ds_left.unsqueeze(0) # ds_right = ds_right.unsqueeze(0) # reproj_left = reproj_left.unsqueeze(0) with torch.no_grad(): output1, output3 = model(ds_left, ds_right) b, h, w = output3.shape b, c, hg, wg = rec_left_gt.shape ds = hg / h diff = torch.abs(output1 - output3) print("diff: ", diff.sum()) output3 = output3.unsqueeze(1) output3 = F.upsample(output3, [hg, wg], mode='bilinear') output3 = torch.mul(output3, ds) output3 = torch.squeeze(output3, 1) depth3 = submodule.reprojection()(output3, reproj_left) gt_z = rec_left_gt[:, 2, :, :] cp_z = depth3[:, 2, :, :] z_mask = mask_left[:, 2, :, :] mae_z = torch.mean(torch.abs(gt_z[z_mask] - cp_z[z_mask])) mae_xyz = torch.mean(torch.abs(rec_left_gt[mask_left] - depth3[mask_left])) bad2_mask = torch.abs(gt_z - cp_z) > 2 bad1_mask = torch.abs(gt_z - cp_z) > 1 bad05_mask = torch.abs(gt_z - cp_z) > 0.5 bad10_mask = torch.abs(gt_z - cp_z) > 10 bad2 = gt_z[bad2_mask & z_mask].numel() / gt_z[z_mask].numel() bad1 = gt_z[bad1_mask & z_mask].numel() / gt_z[z_mask].numel() bad05 = gt_z[bad05_mask & z_mask].numel() / gt_z[z_mask].numel() bad10 = gt_z[bad10_mask & z_mask].numel() / gt_z[z_mask].numel() return mae_xyz, mae_z, bad10, bad2, bad1, bad05, count
def train(sample): model.train() left = sample['left'] # [B, 3, H, W] right = sample['right'] depth = sample['depth'] # [B, H, W] mask = sample['mask'] reproj_left = sample['reprojection'] if args.cuda: left, right, depth, mask, reproj_left = left.cuda(), right.cuda( ), depth.cuda(), mask.cuda(), reproj_left.cuda() # mask = mask > 0.6 b, c, h, w = depth.shape count = b for i in range(b): if depth[i][mask[i]].numel() < depth[i].numel() * 0.1: mask[i] = mask[i] < 0 count -= 1 if count < 1: return -1, -1, -1, -1, -1 optimizer.zero_grad() if args.model == 'GwcNet_GC': output = model(left, right) output0 = output[0] output1 = output[1] output2 = output[2] output3 = output[3] b, h, w = output1.shape bg, c, hg, wg = depth.shape output0 = output0.unsqueeze(1) output1 = output1.unsqueeze(1) output2 = output2.unsqueeze(1) output3 = output3.unsqueeze(1) ds = hg / h output0 = F.upsample(output0, [hg, wg], mode='bilinear') output1 = F.upsample(output1, [hg, wg], mode='bilinear') output2 = F.upsample(output2, [hg, wg], mode='bilinear') output3 = F.upsample(output3, [hg, wg], mode='bilinear') output0 = torch.mul(output0, ds) output1 = torch.mul(output1, ds) # 上采样后要乘以采样率 output2 = torch.mul(output2, ds) output3 = torch.mul(output3, ds) output0 = torch.squeeze(output0, 1) output1 = torch.squeeze(output1, 1) output2 = torch.squeeze(output2, 1) output3 = torch.squeeze(output3, 1) # 输出是batch*height*width depth0 = submodule.reprojection()(output0, reproj_left) depth1 = submodule.reprojection()(output1, reproj_left) depth2 = submodule.reprojection()(output2, reproj_left) depth3 = submodule.reprojection()(output3, reproj_left) # fn = torch.nn.MSELoss() loss = 0.5 * F.smooth_l1_loss( depth0[mask], depth[mask]) + 0.5 * F.smooth_l1_loss( depth1[mask], depth[mask]) + 0.7 * F.smooth_l1_loss( depth2[mask], depth[mask]) + F.smooth_l1_loss( depth3[mask], depth[mask]) elif args.model == 'basic': output3 = model(ds_left, ds_right) output3 = output3.unsqueeze(1) b, d, h, w = output3.shape bg, c, hg, wg = rec_left_gt.shape ds = hg / h output3 = F.upsample(output3, [hg, wg], mode='bilinear') output3 = torch.mul(output3, ds) output3 = torch.squeeze(output3, 1) depth3 = submodule.reprojection()(output3, reproj_left) loss = F.l1_loss(rec_left_gt[mask_left], depth3[mask_left], size_average=True) loss.backward() optimizer.step() # display.display_color_disparity_depth(step, writer, ds_left, output3.unsqueeze(1), depth3, is_return_img=False) return loss.item(), count, output3, depth3, mask
def train(sample): model.train() left = sample['left'] # [B, 3, H, W] right = sample['right'] disp = sample['disp'] # [B, H, W] mask = sample['mask'] if args.cuda: left, right, disp, mask = left.cuda(), right.cuda(), disp.cuda( ), mask.cuda() mask = mask > 0.6 b, c, h, w = disp.shape count = b for i in range(b): if disp[i][mask[i]].numel() < disp[i].numel() * 0.1: mask[i] = mask[i] < 0 count -= 1 if count < 1: return -1, -1, -1 optimizer.zero_grad() if args.use_hourglass == True: output1, output2, output3 = model(left, right) b, h, w = output1.shape bg, c, hg, wg = disp.shape output1 = output1.unsqueeze(1) output2 = output2.unsqueeze(1) output3 = output3.unsqueeze(1) ds = hg / h output1 = F.upsample(output1, [hg, wg], mode='bilinear') output2 = F.upsample(output2, [hg, wg], mode='bilinear') output3 = F.upsample(output3, [hg, wg], mode='bilinear') output1 = torch.mul(output1, ds) # 上采样后要乘以采样率 output2 = torch.mul(output2, ds) output3 = torch.mul(output3, ds) # output1 = torch.squeeze(output1, 1) # output2 = torch.squeeze(output2, 1) # output3 = torch.squeeze(output3, 1) # 输出是batch*height*width # depth1 = submodule.reprojection()(output1, reproj_left) # depth2 = submodule.reprojection()(output2, reproj_left) # depth3 = submodule.reprojection()(output3, reproj_left) # gt = rec_left_gt[0].cpu().numpy() # np.save('gt.npy', gt) # bb = bb[0].cpu().numpy() # np.save('b.npy', bb) # ff = ff[0].cpu().numpy() # np.save('f.npy', ff) # disp = output3[0].cpu().detach().numpy() # np.save('disparity.npy', disp) # z = depth3[0].cpu().detach().numpy() # np.save('depth.npy', z) # fn = torch.nn.MSELoss() # loss = 0.5 * F.smooth_l1_loss(depth1[mask_left], rec_left_gt[mask_left]) + 0.7 * F.smooth_l1_loss(depth2[mask_left], rec_left_gt[mask_left]) + F.smooth_l1_loss(depth3[mask_left], rec_left_gt[mask_left]) loss = 0.5 * F.smooth_l1_loss( output1[mask], disp[mask]) + 0.7 * F.smooth_l1_loss( output2[mask], disp[mask]) + F.smooth_l1_loss( output3[mask], disp[mask]) elif args.model == 'basic': output3 = model(ds_left, ds_right) output3 = output3.unsqueeze(1) b, d, h, w = output3.shape bg, c, hg, wg = rec_left_gt.shape ds = hg / h output3 = F.upsample(output3, [hg, wg], mode='bilinear') output3 = torch.mul(output3, ds) output3 = torch.squeeze(output3, 1) depth3 = submodule.reprojection()(output3, reproj_left) loss = F.l1_loss(rec_left_gt[mask_left], depth3[mask_left], size_average=True) loss.backward() optimizer.step() # display.display_color_disparity_depth(step, writer, ds_left, output3.unsqueeze(1), depth3, is_return_img=False) return loss.item(), count, output3