def write_vls_eval(self, data_blob, outputs, tagname, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) inputdepth = tensor2disp(1 / data_blob['depthmap'], vmax=1, viewind=0) depthpredvls = tensor2disp(1 / outputs[('depth', 2)], vmax=1, viewind=0) flowvls = flow_to_image( outputs[('flowpred', 2)][0].detach().cpu().permute([1, 2, 0]).numpy(), rad_max=10) imgrecon = tensor2rgb(outputs[('reconImg', 2)], viewind=0) img_val_up = np.concatenate([np.array(img1), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(inputdepth), np.array(depthpredvls)], axis=1) img_val_mid3 = np.concatenate( [np.array(imgrecon), np.array(flowvls)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid2), np.array(img_val_mid3) ], axis=0) self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(img1, img2, data_blob['depthmap'], data_blob['flowpred'], outputs) self.writer.add_image('{}_X'.format(tagname), (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def write_vls(self, data_blob, outputs, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'], vmax=0.15, viewind=0) imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0) attentionmap = tensor2disp(outputs['attention'], percentile=95, viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid = np.concatenate( [np.array(depthpredvls), np.array(attentionmap)], axis=1) img_val = np.concatenate( [np.array(img_val_up), np.array(img_val_mid)], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], outputs) self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def write_vls(self, data_blob, outputs, flowselector, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() figmask_flow = tensor2disp(flowselector, vmax=1, viewind=0) insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs[('depth', 2)], vmax=0.15, viewind=0) flowvls = flow_to_image( outputs[('flowpred', 2)][0].detach().cpu().permute([1, 2, 0]).numpy(), rad_max=10) imgrecon = tensor2rgb(outputs[('reconImg', 2)], viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(figmask_flow)], axis=1) img_val_mid3 = np.concatenate( [np.array(imgrecon), np.array(flowvls)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid2), np.array(img_val_mid3) ], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step)
def write_vls_eval(self, data_blob, outputs, tagname, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'], vmax=0.15, viewind=0) imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid = np.concatenate([np.array(depthpredvls), np.array(imgrecon)], axis=1) img_val = np.concatenate([np.array(img_val_up), np.array(img_val_mid)], axis=0) self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step)
def write_vls(self, data_blob, outputs, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'], vmax=0.15, viewind=0) imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0) flow_RAFT = Image.fromarray( flow_to_image(data_blob['flowpred'][0].permute([1, 2, 0]).cpu().numpy())) flow_pred = Image.fromarray( flow_to_image(outputs[('flowpred', 2)][0].permute([1, 2, 0]).detach().cpu().numpy())) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid = np.concatenate( [np.array(depthpredvls), np.array(imgrecon)], axis=1) img_val_flow = np.concatenate( [np.array(flow_RAFT), np.array(flow_pred)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid), np.array(img_val_flow) ], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], outputs) self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step) X = self.vls_objmvment(np.array(insvls), data_blob['insmap'], data_blob['posepred']) self.writer.add_image('objmvment', (torch.from_numpy(X).float() / 255).permute([2, 0, 1]), step)
def write_vls_eval(self, data_blob, outputs, tagname, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs['depth_predictions'][-1], vmax=0.15, viewind=0) depthgtvls = tensor2disp(1 / data_blob['depthmap'], vmax=0.15, viewind=0) flowvls = flow_to_image(outputs['flowpred'][0].detach().cpu().permute( [1, 2, 0]).numpy(), rad_max=50) imgrecon = tensor2rgb(outputs['img1_recon'], viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(depthgtvls)], axis=1) img_val_mid3 = np.concatenate( [np.array(imgrecon), np.array(flowvls)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid2), np.array(img_val_mid3) ], axis=0) self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], data_blob['flowmap'], data_blob['insmap'], outputs) self.writer.add_image('{}_X'.format(tagname), (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def validate_kitti_colorjitter(gpu, model, args, ngpus_per_node, eval_entries, iters=24): """ Peform validation using the KITTI-2015 (train) split """ interval = np.floor(len(eval_entries) / ngpus_per_node).astype( np.int).item() if gpu == ngpus_per_node - 1: stidx = int(interval * gpu) edidx = len(eval_entries) else: stidx = int(interval * gpu) edidx = int(interval * (gpu + 1)) print("Initialize Instance on Gpu %d, from %d to %d, total %d" % (gpu, stidx, edidx, len(eval_entries))) from tqdm import tqdm model.eval() model.cuda(gpu) with torch.no_grad(): for val_id, entry in enumerate( tqdm(remove_dup(eval_entries[stidx:edidx]))): seq, index = entry.split(' ') index = int(index) img1path = os.path.join(args.dataset, seq, 'rgb_{}.png'.format(str(index).zfill(5))) img2path = os.path.join( args.dataset, seq, 'rgb_{}.png'.format(str(index + 1).zfill(5))) if not os.path.exists(img2path) and not os.path.exists( img2path.replace('.png', '.jpg')): img2path = img1path if not os.path.exists(img1path): img1path = img1path.replace('.png', '.jpg') img2path = img2path.replace('.png', '.jpg') image1 = frame_utils.read_gen(img1path) image2 = frame_utils.read_gen(img2path) image1 = np.array(image1).astype(np.uint8) image2 = np.array(image2).astype(np.uint8) image1 = torch.from_numpy(image1).permute([2, 0, 1]).float() image2 = torch.from_numpy(image2).permute([2, 0, 1]).float() svfold = os.path.join( args.exportroot, seq, ) svpath = os.path.join(args.exportroot, seq, '{}.png'.format(str(index).zfill(5))) os.makedirs(svfold, exist_ok=True) image1 = image1[None].cuda(gpu) image2 = image2[None].cuda(gpu) flow_low, flow_pr = model(image1, image2, iters=iters, test_mode=True) flow = flow_pr.squeeze(0) flow_numpy = flow.cpu().permute(1, 2, 0).numpy() frame_utils.writeFlowKITTI(svpath, flow_numpy) if args.dovls: vlsflow = Image.fromarray(flow_viz.flow_to_image(flow_numpy)) vlsrgb1 = tensor2rgb(image1 / 255.0, viewind=0) vlsrgb2 = tensor2rgb(image2 / 255.0, viewind=0) w, h = vlsrgb2.size xx, yy = np.meshgrid(range(w), range(h), indexing='xy') xxf = xx.flatten() yyf = yy.flatten() rndidx = np.random.randint(0, xxf.shape[0], 100) xxf_rnd = xxf[rndidx] yyf_rnd = yyf[rndidx] flowx = flow_numpy[yyf_rnd, xxf_rnd, 0] flowy = flow_numpy[yyf_rnd, xxf_rnd, 1] fig = plt.figure(figsize=(12, 9)) plt.subplot(2, 1, 1) plt.scatter(xxf_rnd, yyf_rnd, 1, 'r') plt.imshow(vlsrgb1) plt.subplot(2, 1, 2) plt.scatter(flowx + xxf_rnd, flowy + yyf_rnd, 1, 'r') plt.imshow(vlsrgb2) plt.savefig( os.path.join( "/media/shengjie/disk1/Prediction/nyuv2_flow_vls", "{}_{}.jpg".format(seq, str(index).zfill(5)))) plt.close() # combined_left = np.concatenate([np.array(vlsrgb1), np.array(vlsrgb2)], axis=0) # combined_right = np.concatenate([np.array(vlsflow), np.array(vlsflow)], axis=0) # combined = np.concatenate([combined_left, combined_right], axis=1) # vls_sv_root = os.makedirs("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", exist_ok=True) # Image.fromarray(combined).save(os.path.join("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", "{}_{}.jpg".format(seq, str(index).zfill(5)))) return
def validate_RANSAC_scalecontrast(args, eval_loader, group, num_frames=5, dovls=False): gpu = args.gpu eval_metrics = { 'pose_RANSAC_mdscale': torch.zeros(1).cuda(device=gpu), 'pose_RANSAC': torch.zeros(num_frames).cuda(device=gpu), 'pixelnum': torch.zeros(1).cuda(device=gpu), 'framenum': torch.zeros(1).cuda(device=gpu) } for val_id, data_blob in enumerate(tqdm(eval_loader)): image1 = data_blob['img1'].cuda(gpu) image2 = data_blob['img2'].cuda(gpu) insmap = data_blob['insmap'].cuda(gpu) intrinsic = data_blob['intrinsic'].cuda(gpu) depthgt = data_blob['depthmap'].cuda(gpu) valid_flow = data_blob['valid_flow'].cuda(gpu) == 1 tagname = data_blob['tag'][0] flowgt_stereo = data_blob['flowgt_stereo'].cuda(gpu) posepred_deepv2d = data_blob['posepred_deepv2d'].cuda(gpu) flowpred = data_blob['flowpred'].cuda(gpu) # mdDepth_pred = data_blob['mdDepth_pred'].cuda(gpu) mdDepth_pred = data_blob['depthpred_deepv2d'].cuda(gpu) mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt() val = (mag < MAX_FLOW) * (insmap == 0) * valid_flow * (depthgt > 0) # R, t, scale, scale_md, flow_sel_mag, istatic = inf_pose_flow(image1, image2, flowpred, insmap, depthgt, mdDepth_pred, intrinsic, int(val_id + args.gpu * 100000), gradComputer=None) R, t, scale, scale_md, flow_sel_mag, istatic = inf_pose_flow( image1, image2, flowpred, insmap, depthgt, mdDepth_pred, intrinsic, int(val_id + args.gpu * 1000 + 10), gradComputer=None) pose_RANSAC = torch.clone(posepred_deepv2d) pose_RANSAC_mdscale = torch.clone(posepred_deepv2d) if istatic: pose_RANSAC[0, 0, :, :] = torch.eye(4, device=intrinsic.device) pose_RANSAC_mdscale[0, 0, :, :] = torch.eye(4, device=intrinsic.device) else: pose_RANSAC[0, 0, 0:3, 0:3] = torch.from_numpy(R).float().cuda( intrinsic.device) pose_RANSAC[0, 0, 0:3, 3:4] = torch.from_numpy( t * scale).float().cuda(intrinsic.device) pose_RANSAC_mdscale[0, 0, 0:3, 0:3] = torch.from_numpy(R).float().cuda( intrinsic.device) pose_RANSAC_mdscale[0, 0, 0:3, 3:4] = torch.from_numpy( t * scale_md).float().cuda(intrinsic.device) poses = { 'pose_RANSAC_mdscale': pose_RANSAC_mdscale, 'pose_RANSAC': pose_RANSAC } if dovls: valnp = val.squeeze().cpu().numpy() == 1 fig, axs = plt.subplots(3, 1, figsize=(16, 9)) rgbvls = tensor2rgb(image1 / 255.0, viewind=0) axs[2].imshow(tensor2rgb(image2 / 255.0, viewind=0)) for idx, k in enumerate(poses.keys()): flow_eval = depth2flow(depthgt, depthgt > 0, intrinsic, poses[k]) epe = torch.sum((flowgt_stereo - flow_eval)**2, dim=1, keepdim=True).sqrt() mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt() out = ((epe[val] > 3.0) & ((epe[val] / mag[val]) > 0.05)).float() eval_metrics[k][0] += torch.sum(out) if dovls: _, _, h, w = depthgt.shape xx, yy = np.meshgrid(range(w), range(h), indexing='xy') xxval = xx[valnp] yyval = yy[valnp] outnp = out.cpu().numpy() == 1 if k == 'pose_RANSAC': titlestr = "%s, Performance Level %f, scale_gt: %.4f" % ( k, (torch.sum(out).item() / out.shape[0]), scale) else: titlestr = "%s, Performance Level %f, reldiff: %.4f" % ( k, (torch.sum(out).item() / out.shape[0]), np.abs(scale_md - scale) / scale) axs[idx].scatter(xxval, yyval, 0.01, 'b') axs[idx].scatter(xxval[outnp], yyval[outnp], 1, 'r') axs[idx].imshow(rgbvls) axs[idx].title.set_text(titlestr) if dovls: os.makedirs(args.vlsroot, exist_ok=True) plt.savefig(os.path.join(args.vlsroot, "{}.png".format(tagname.split('/')[0])), bbox_inches='tight', pad_inches=0) plt.close() eval_metrics['pixelnum'][0] += torch.sum(val) eval_metrics['framenum'][0] += 1 if args.distributed: for k in eval_metrics.keys(): dist.all_reduce(tensor=eval_metrics[k], op=dist.ReduceOp.SUM, group=group) if args.gpu == 0: print_keys = ['pose_RANSAC_mdscale', 'pose_RANSAC'] print_str = "Evaluate Poses on Full Split %d samples" % ( eval_metrics['framenum'][0].item()) for key in print_keys: print_str += ", %s-out: %f" % (key, eval_metrics[key][0].item() / eval_metrics['pixelnum'][0].item()) print(print_str)
def validate_kitti(model, args, eval_loader, group, isdeepv2d=False): """ Peform validation using the KITTI-2015 (train) split """ """ Peform validation using the KITTI-2015 (train) split """ model.eval() gpu = args.gpu eval_measures_depth = torch.zeros(10).cuda(device=gpu) vlsroot = '/media/shengjie/disk1/visualization/paper_depthvls' residual_root = os.path.join(vlsroot, 'residual_vls') bts_root = os.path.join(vlsroot, 'bts_vls') ours_root = os.path.join(vlsroot, 'ours_vls') deepv2d_root = os.path.join(vlsroot, 'deepv2d_vls') rgb_root = os.path.join(vlsroot, 'rgb_in') # os.makedirs(residual_root, exist_ok=True) # os.makedirs(bts_root, exist_ok=True) # os.makedirs(ours_root, exist_ok=True) # os.makedirs(deepv2d_root, exist_ok=True) # os.makedirs(rgb_root, exist_ok=True) for val_id, data_blob in enumerate(tqdm(eval_loader)): image1 = data_blob['img1'].cuda(gpu) / 255.0 image2 = data_blob['img2'].cuda(gpu) / 255.0 intrinsic = data_blob['intrinsic'].cuda(gpu) insmap = data_blob['insmap'].cuda(gpu) posepred = data_blob['posepred'].cuda(gpu) depthgt = data_blob['depthmap'].cuda(gpu) if not args.initbymD: mD_pred = data_blob['depthpred'].cuda(gpu) else: mD_pred = data_blob['mdDepth_pred'].cuda(gpu) svname = "{}.png".format(str(val_id).zfill(10)) mD_pred_clipped = torch.clamp_min(mD_pred, min=args.min_depth_pred) outputs = model(image1, image2, mD_pred_clipped, intrinsic, posepred, insmap) predread = outputs[('depth', 2)] depthpred_deepv2d = data_blob['depthpred_deepv2d'].cuda(gpu) sigmoidact = outputs[('residualdepth', 2)] # tensor2disp(1 / mD_pred_clipped, vmax=0.15, viewind=0).save(os.path.join(bts_root, svname)) # tensor2disp(1 / predread, vmax=0.15, viewind=0).save(os.path.join(ours_root, svname)) # tensor2disp(1 / depthpred_deepv2d, vmax=0.15, viewind=0).save(os.path.join(deepv2d_root, svname)) # tensor2rgb(image1, viewind=0).save(os.path.join(rgb_root, svname)) # tensor2grad(sigmoidact, pos_bar=0.1, neg_bar=-0.1, viewind=0).save(os.path.join(residual_root, svname)) fig1 = tensor2rgb(image1, viewind=0) fig1_2 = tensor2rgb(image2, viewind=0) fig2 = tensor2disp(1 / depthpred_deepv2d, vmax=0.15, viewind=0) fig3 = tensor2disp(1 / mD_pred_clipped, vmax=0.15, viewind=0) fig4 = tensor2grad(sigmoidact, pos_bar=0.1, neg_bar=-0.1, viewind=0) fig5 = tensor2disp(1 / predread, vmax=0.15, viewind=0) figs = concat_imgs([fig1, fig1_2, fig2, fig3, fig4, fig5]) figc1 = np.concatenate([np.array(figs[0]), np.array(figs[1])], axis=0) figc2 = np.concatenate([np.array(figs[4]), np.array(figs[2])], axis=0) figc3 = np.concatenate([np.array(figs[3]), np.array(figs[5])], axis=0) imgvls = np.concatenate([figc1, figc2, figc3], axis=1) imgvls = Image.fromarray(imgvls) imgvls.save(os.path.join(vlsroot, svname)) selector = ((depthgt > 0) * (predread > 0) * (depthgt > args.min_depth_eval) * (depthgt < args.max_depth_eval)).float() predread = torch.clamp(predread, min=args.min_depth_eval, max=args.max_depth_eval) depth_gt_flatten = depthgt[selector == 1].cpu().numpy() pred_depth_flatten = predread[selector == 1].cpu().numpy() pred_depth_flatten = np.median(depth_gt_flatten/pred_depth_flatten) * pred_depth_flatten eval_measures_depth_np = compute_errors(gt=depth_gt_flatten, pred=pred_depth_flatten) eval_measures_depth[:9] += torch.tensor(eval_measures_depth_np).cuda(device=gpu) eval_measures_depth[9] += 1 if args.distributed: dist.all_reduce(tensor=eval_measures_depth, op=dist.ReduceOp.SUM, group=group) if args.gpu == 0: eval_measures_depth[0:9] = eval_measures_depth[0:9] / eval_measures_depth[9] eval_measures_depth = eval_measures_depth.cpu().numpy() print('Computing Depth errors for %f eval samples' % (eval_measures_depth[9].item())) print("{:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}".format('silog', 'abs_rel', 'log10', 'rms', 'sq_rel', 'log_rms', 'd1', 'd2', 'd3')) for i in range(8): print('{:7.3f}, '.format(eval_measures_depth[i]), end='') print('{:7.3f}'.format(eval_measures_depth[8])) return {'silog': float(eval_measures_depth[0]), 'abs_rel': float(eval_measures_depth[1]), 'log10': float(eval_measures_depth[2]), 'rms': float(eval_measures_depth[3]), 'sq_rel': float(eval_measures_depth[4]), 'log_rms': float(eval_measures_depth[5]), 'd1': float(eval_measures_depth[6]), 'd2': float(eval_measures_depth[7]), 'd3': float(eval_measures_depth[8]) } else: return None
def validate_RANSAC_odom_relpose(args, eval_loader, banins=False, bangrad=False, samplenum=50000): if bangrad: gradComputer = None else: gradComputer = GradComputer() for val_id, data_blob in enumerate(tqdm(eval_loader)): insmap = data_blob['insmap'] intrinsic = data_blob['intrinsic'] flowpred = data_blob['flowpred_RAFT'] mdDepth_pred = data_blob['mdDepth'] pose_bs = data_blob['posepred_bs'] tag = data_blob['tag'][0] if torch.sum(torch.abs(data_blob['img1'] - data_blob['img2'])) < 1: R = np.eye(3) t = np.array([[0, 0, -1]]).T scale = 0 else: R, t, scale, _ = inf_pose_flow(flowpred, insmap, mdDepth_pred, intrinsic, int(val_id + 10), gradComputer=gradComputer, banins=banins, samplenum=samplenum) self_pose = np.eye(4) self_pose[0:3, 0:3] = R self_pose[0:3, 3:4] = t * scale pose_bs_np = pose_bs[0].cpu().numpy() pose_bs_np = pose_bs_np @ np.linalg.inv(pose_bs_np[0]) @ self_pose pose_bs_np[0] = self_pose seq, frmidx = tag.split(' ') exportfold = os.path.join(args.vls_root, seq, 'image_02') insmap_np = data_blob['insmap'].squeeze().numpy() xx, yy = np.meshgrid(range(insmap_np.shape[1]), range(insmap_np.shape[0]), indexing='xy') fig, ax = plt.subplots(figsize=(16,9)) ax.imshow(vls_ins(np.array(tensor2rgb(data_blob['img1'], viewind=0)), insmap_np)) for k in np.unique(insmap_np): if k == 0: continue xxf = xx[insmap_np == k] yyf = yy[insmap_np == k] xmin = xxf.min() xmax = xxf.max() ymin = yyf.min() ymax = yyf.max() if (ymax - ymin) * (xmax - xmin) < 1000: continue rect = patches.Rectangle((xmin, ymax), xmax - xmin, ymin - ymax, linewidth=1, facecolor='none', edgecolor='r') ax.add_patch(rect) ins_relpose = pose_bs[0, k].cpu().numpy() @ np.linalg.inv(pose_bs[0, 0].cpu().numpy()) mvdist = np.sqrt(np.sum(ins_relpose[0:3, 3:4] ** 2)) mvdist_transed = np.sqrt(np.sum((pose_bs_np[k] @ np.linalg.inv(pose_bs_np[0]))[0:3, 3:4] ** 2)) assert np.abs(mvdist_transed - mvdist) < 1e-3 ax.text(xmin + 5, ymin + 10, '%.3f' % mvdist, fontsize=6, c='r', weight='bold') plt.axis('off') os.makedirs(exportfold, exist_ok=True) plt.savefig(os.path.join(exportfold, "{}.png".format(frmidx)), bbox_inches='tight', pad_inches=0) plt.close()
def validate_RANSAC_poses(args, eval_loader, group, num_frames=5, dovls=False, gtscale=True): gpu = args.gpu eval_metrics = { 'deepv2d_pose': torch.zeros(1).cuda(device=gpu), 'pose_RANSAC': torch.zeros(num_frames).cuda(device=gpu), 'pixelnum': torch.zeros(1).cuda(device=gpu), 'framenum': torch.zeros(1).cuda(device=gpu) } gradComputer = GradComputer() for val_id, data_blob in enumerate(tqdm(eval_loader)): image1 = data_blob['img1'].cuda(gpu) image2 = data_blob['img2'].cuda(gpu) insmap = data_blob['insmap'].cuda(gpu) intrinsic = data_blob['intrinsic'].cuda(gpu) depthgt = data_blob['depthmap'].cuda(gpu) valid_flow = data_blob['valid_flow'].cuda(gpu) == 1 tagname = data_blob['tag'][0] flowgt_stereo = data_blob['flowgt_stereo'].cuda(gpu) posepred_deepv2d = data_blob['posepred_deepv2d'].cuda(gpu) flowpred = data_blob['flowpred'].cuda(gpu) mdDepth_pred = data_blob['mdDepth_pred'].cuda(gpu) # mdDepth_pred = data_blob['depthpred_deepv2d'].cuda(gpu) mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt() val = (mag < MAX_FLOW) * (insmap == 0) * valid_flow * (depthgt > 0) if not gtscale: R, t, scale, flow_sel_mag, istatic, insdict = inf_pose_flow( image1, image2, flowpred, insmap, mdDepth_pred, intrinsic, int(val_id + args.gpu * 100000), gradComputer=gradComputer) else: R, t, scale, flow_sel_mag, istatic, insdict = inf_pose_flow( image1, image2, flowpred, insmap, depthgt, intrinsic, int(val_id + args.gpu * 100000), gradComputer=None) pose_RANSAC = torch.clone(posepred_deepv2d) if istatic: pose_RANSAC[0, 0, :, :] = torch.eye(4, device=intrinsic.device) else: pose_RANSAC[0, 0, 0:3, 0:3] = torch.from_numpy(R).float().cuda( intrinsic.device) pose_RANSAC[0, 0, 0:3, 3:4] = torch.from_numpy( t * scale).float().cuda(intrinsic.device) poses = {'deepv2d_pose': posepred_deepv2d, 'pose_RANSAC': pose_RANSAC} if dovls: valnp = val.squeeze().cpu().numpy() == 1 fig, axs = plt.subplots(3, 1, figsize=(16, 9)) rgbvls = tensor2rgb(image1 / 255.0, viewind=0) axs[2].imshow(tensor2rgb(image2 / 255.0, viewind=0)) for idx, k in enumerate(poses.keys()): flow_eval = depth2flow(depthgt, depthgt > 0, intrinsic, poses[k]) epe = torch.sum((flowgt_stereo - flow_eval)**2, dim=1, keepdim=True).sqrt() mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt() out = ((epe[val] > 3.0) & ((epe[val] / mag[val]) > 0.05)).float() eval_metrics[k][0] += torch.sum(out) if dovls: _, _, h, w = depthgt.shape xx, yy = np.meshgrid(range(w), range(h), indexing='xy') xxval = xx[valnp] yyval = yy[valnp] outnp = out.cpu().numpy() == 1 titlestr = "%s, Performance Level %f, flows_mag %f" % (k, ( torch.sum(out).item() / out.shape[0]), flow_sel_mag) if istatic: titlestr += ", static" else: titlestr += ", moving" axs[idx].scatter(xxval, yyval, 0.01, 'b') axs[idx].scatter(xxval[outnp], yyval[outnp], 1, 'r') if k == 'pose_RANSAC': insmap_np = insmap.squeeze().cpu().numpy() for insid in np.unique(insmap_np): if insid == 0: continue if insid not in insdict.keys(): continue insdata = insdict[insid] rect = patches.Rectangle(insdata['bbox'][0], insdata['bbox'][1], insdata['bbox'][2], linewidth=1, edgecolor='r', facecolor='none') axs[idx].add_patch(rect) axs[idx].text(insdata['bbox'][0][0] + 5, insdata['bbox'][0][1] + insdata['bbox'][2] + 20, '%.3f' % (insdata['ave_dist']), fontsize=10, c='r', weight='bold') axs[idx].imshow(rgbvls) axs[idx].title.set_text(titlestr) if dovls: os.makedirs(args.vlsroot, exist_ok=True) plt.savefig(os.path.join(args.vlsroot, "{}.png".format(tagname.split('/')[0])), bbox_inches='tight', pad_inches=0) plt.close() eval_metrics['pixelnum'][0] += torch.sum(val) eval_metrics['framenum'][0] += 1 if args.distributed: for k in eval_metrics.keys(): dist.all_reduce(tensor=eval_metrics[k], op=dist.ReduceOp.SUM, group=group) if args.gpu == 0: print_keys = ['deepv2d_pose', 'pose_RANSAC'] print_str = "Evaluate Poses on Full Split %d samples" % ( eval_metrics['framenum'][0].item()) for key in print_keys: print_str += ", %s-out: %f" % (key, eval_metrics[key][0].item() / eval_metrics['pixelnum'][0].item()) print(print_str)
def validate_RANSAC_odom_relpose(args, eval_loader, banins=False, bangrad=False, samplenum=50000): if bangrad: gradComputer = None else: gradComputer = GradComputer() ssim = SSIM() trialtime = 32 os.makedirs(args.vls_root, exist_ok=True) for val_id, data_blob in enumerate(tqdm(eval_loader)): insmap = data_blob['insmap'] intrinsic = data_blob['intrinsic'] flowpred = data_blob['flowpred_RAFT'] mdDepth_pred = data_blob['mdDepth'] tag = data_blob['tag'][0] flowgt = data_blob['flowgt'] rgb1 = data_blob['img1'] / 255.0 rgb2 = data_blob['img2'] / 255.0 insselector = (insmap == 0).float() stereogt = data_blob['stereogt'] flowgt_valid = data_blob['flowgt_valid'] * insselector * (stereogt > 0).float() self_poses = list() breakflag = False for k in range(trialtime): R, t, scale, _ = inf_pose_flow(flowpred, insmap, mdDepth_pred, intrinsic, int(val_id * 500 + k), gradComputer=gradComputer, banins=banins, samplenum=samplenum) if scale == 0: breakflag = True else: self_pose = np.eye(4) self_pose[0:3, 0:3] = R self_pose[0:3, 3:4] = t * scale self_pose = torch.from_numpy(self_pose).float() self_poses.append(self_pose) if breakflag: continue rpj_loss_rec = list() flow_loss_rec = list() for k in range(trialtime): self_pose = self_poses[k] mdDepth_pred_sqz = mdDepth_pred.squeeze() _, _, h, w = insmap.shape xx, yy = np.meshgrid(range(w), range(h), indexing='xy') xxt = torch.from_numpy(xx) yyt = torch.from_numpy(yy) ones = torch.ones_like(xxt) pts3d = torch.stack([xxt * mdDepth_pred_sqz, yyt * mdDepth_pred_sqz, mdDepth_pred_sqz, ones], dim=2).unsqueeze(-1) intrinsic_sqz = intrinsic.squeeze() projM = intrinsic_sqz @ self_pose @ torch.inverse(intrinsic_sqz) projM_ex = projM.unsqueeze(0).unsqueeze(0).expand([h, w, -1, -1]) prj2d = projM_ex @ pts3d prj2d_x = prj2d[:, :, 0] / prj2d[:, :, 2] prj2d_y = prj2d[:, :, 1] / prj2d[:, :, 2] flowx = prj2d_x.squeeze() - xxt flowy = prj2d_y.squeeze() - yyt flowpred_depth = torch.stack([flowx, flowy], dim=0).unsqueeze(0) # Image.fromarray(flow_to_image(flowpred_depth[0].permute([1, 2, 0]).numpy(), rad_max=300)).show() # Image.fromarray(flow_to_image(flowpred[0].permute([1, 2, 0]).numpy(), rad_max=300)).show() prj2d_x = (prj2d_x / (w - 1) - 0.5) * 2 prj2d_y = (prj2d_y / (h - 1) - 0.5) * 2 prj2dpx = torch.cat([prj2d_x, prj2d_y], dim=-1).unsqueeze(0) rgb1_recon = F.grid_sample(rgb2, prj2dpx, mode='bilinear', align_corners=False) rpj_loss, rpj_loss_imgsize = reprojetion_loss(rgb1, rgb1_recon, ssim, insselector) mdDepth_pred_sqz = stereogt.squeeze() _, _, h, w = insmap.shape xx, yy = np.meshgrid(range(w), range(h), indexing='xy') xxt = torch.from_numpy(xx) yyt = torch.from_numpy(yy) ones = torch.ones_like(xxt) pts3d = torch.stack([xxt * mdDepth_pred_sqz, yyt * mdDepth_pred_sqz, mdDepth_pred_sqz, ones], dim=2).unsqueeze(-1) intrinsic_sqz = intrinsic.squeeze() projM = intrinsic_sqz @ self_pose @ torch.inverse(intrinsic_sqz) projM_ex = projM.unsqueeze(0).unsqueeze(0).expand([h, w, -1, -1]) prj2d = projM_ex @ pts3d prj2d_x = prj2d[:, :, 0] / prj2d[:, :, 2] prj2d_y = prj2d[:, :, 1] / prj2d[:, :, 2] flowx = prj2d_x.squeeze() - xxt flowy = prj2d_y.squeeze() - yyt flowpred_depth = torch.stack([flowx, flowy], dim=0).unsqueeze(0) flow_loss = torch.sum(torch.abs(flowpred_depth - flowgt), dim=1, keepdim=True) flow_loss_imgsize = flow_loss * flowgt_valid flow_loss = torch.sum(flow_loss * flowgt_valid) / torch.sum(flowgt_valid) # tensor2rgb(rgb1_recon, viewind=0).show() rpj_loss_rec.append(rpj_loss.item()) flow_loss_rec.append(flow_loss.item()) if k == 0: fig = plt.figure(figsize=(16, 9)) fig.add_subplot(4, 1, 1) plt.imshow(tensor2rgb(rgb1, viewind=0)) fig.add_subplot(4, 1, 2) plt.imshow(tensor2disp(rpj_loss_imgsize, vmax=0.1)) fig.add_subplot(4, 1, 3) plt.imshow(tensor2disp(flow_loss_imgsize, vmax=3)) fig.add_subplot(4, 1, 4) plt.scatter(flow_loss_rec, rpj_loss_rec) plt.xlabel('flow loss') plt.ylabel('photometric loss') svpath = os.path.join(args.vls_root, tag.split(' ')[0].split('/')[1] + '.png') plt.savefig(svpath, bbox_inches='tight') plt.close()
def validate_RANSAC_odom_relpose(args, eval_loader, samplenum=50000, iters=0): gradComputer = GradComputer() # optimized = 0 # skipped = 0 # unresolved = 0 bestids = list() for val_id, data_blob in enumerate(tqdm(eval_loader)): intrinsic = data_blob['intrinsic'] flowpred = data_blob['flowpred_RAFT'] mdDepth_pred = data_blob['mdDepth'] # mdDepth_pred = data_blob['gtDepth'] tag = data_blob['tag'][0] seq, frmidx = tag.split(' ') exportfold = os.path.join(args.export_root, str(iters).zfill(3), seq) os.makedirs(exportfold, exist_ok=True) export_root = os.path.join(exportfold, frmidx.zfill(5) + '.pickle') if torch.sum(torch.abs(data_blob['img1'] - data_blob['img2'])) < 1: R = np.eye(3) t = np.array([[0, 0, -1]]).T scale = 1e-10 bestid = -1 else: R, t, scale, pts1_inliers, pts2_inliers, bestid = inf_pose_flow( flowpred, mdDepth_pred, intrinsic, int(iters * eval_loader.__len__() + val_id), gradComputer=gradComputer, samplenum=samplenum, ban_scale_check=args.ban_scale_check) self_pose = np.eye(4) self_pose[0:3, 0:3] = R self_pose[0:3, 3:4] = t * scale bestids.append(bestid) with open(export_root, 'wb') as handle: pickle.dump(self_pose, handle, protocol=pickle.HIGHEST_PROTOCOL) # if bestid == -1: flow_numpy = flowpred[0].cpu().permute([1, 2, 0]).numpy() _, _, h, w = mdDepth_pred.shape xx, yy = np.meshgrid(range(w), range(h), indexing='xy') xxf = xx.flatten() yyf = yy.flatten() rndidx = np.random.randint(0, xxf.shape[0], 100) xxf_rnd = xxf[rndidx] yyf_rnd = yyf[rndidx] selector = mdDepth_pred.squeeze().cpu().numpy()[yyf_rnd, xxf_rnd] > 0 xxf_rnd = xxf_rnd[selector] yyf_rnd = yyf_rnd[selector] flowx = flow_numpy[yyf_rnd, xxf_rnd, 0] flowy = flow_numpy[yyf_rnd, xxf_rnd, 1] depthf = mdDepth_pred.squeeze().cpu().numpy()[yyf_rnd, xxf_rnd] intrinsic_np = intrinsic.squeeze().numpy() pts3d = np.stack([ xxf_rnd * depthf, yyf_rnd * depthf, np.ones_like(yyf_rnd) * depthf, np.ones_like(yyf_rnd) ], axis=0) pts3d = intrinsic_np @ self_pose @ np.linalg.inv(intrinsic_np) @ pts3d pts3d[0, :] = pts3d[0, :] / pts3d[2, :] pts3d[1, :] = pts3d[1, :] / pts3d[2, :] image1 = data_blob['img1'] image2 = data_blob['img2'] vlsrgb1 = tensor2rgb(image1 / 255.0, viewind=0) vlsrgb2 = tensor2rgb(image2 / 255.0, viewind=0) fig = plt.figure(figsize=(12, 9)) plt.subplot(2, 1, 1) plt.scatter(xxf_rnd, yyf_rnd, 5, 'g') plt.imshow(vlsrgb1) plt.axis('off') plt.subplot(2, 1, 2) plt.scatter(flowx + xxf_rnd, flowy + yyf_rnd, 5, 'r') plt.scatter(pts3d[0, :], pts3d[1, :], 5, 'b') plt.axis('off') # plt.legend(['RAFT Flow', 'Initial Depth + Intial Pose']) plt.imshow(vlsrgb2) # plt.show() plt.savefig( os.path.join("/media/shengjie/disk1/visualization/nyu_pose_vls", "{}_{}.jpg".format(seq, str(frmidx).zfill(5))), bbox_inches='tight', pad_inches=0, ) plt.close()