def write_vls(self, data_blob, outputs, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'], vmax=0.15, viewind=0) imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0) attentionmap = tensor2disp(outputs['attention'], percentile=95, viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid = np.concatenate( [np.array(depthpredvls), np.array(attentionmap)], axis=1) img_val = np.concatenate( [np.array(img_val_up), np.array(img_val_mid)], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], outputs) self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def write_vls_eval(self, data_blob, outputs, tagname, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs[('mDepth', 0)], vmax=0.15, viewind=0) flowvls = flow_to_image( outputs[('flowpred', 0)][0].detach().cpu().permute([1, 2, 0]).numpy(), rad_max=10) imgrecon = tensor2rgb(outputs[('reconImg', 0)], ind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(flowvls)], axis=1) img_val_mid3 = np.concatenate( [np.array(img1), np.array(imgrecon)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid2), np.array(img_val_mid3) ], axis=0) self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(img1, img2, data_blob['depthvls'], outputs) self.writer.add_image('{}_X'.format(tagname), (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def get_vls_eval(data_blob, outputs, args): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs[('depth', 2)], vmax=0.15, viewind=0) depthpredvls_org = tensor2disp(1 / data_blob['depthpred'], vmax=0.15, viewind=0) sigmoidact = outputs[('residualdepth', 2)] sigmoidactvls = tensor2grad(sigmoidact, pos_bar=0.1, neg_bar=-0.1, viewind=0) img_val_up = np.concatenate( [np.array(insvls), np.array(sigmoidactvls)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(depthpredvls_org)], axis=1) img_val = np.concatenate( [np.array(img_val_up), np.array(img_val_mid2)], axis=0) svtag = data_blob['tag'][0] svroot = os.path.join(args.svroot, args.restore_ckpt.split('/')[-2]) os.makedirs(svroot, exist_ok=True) seq, imgname = svtag.split(' ') figname = "{}_{}".format(seq.split('/')[-1], imgname) vlsname = os.path.join(svroot, "{}.png".format(figname)) Image.fromarray(img_val).save(vlsname)
def write_vls_eval(self, data_blob, outputs, tagname, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs[('depth', 2)], vmax=0.15, viewind=0) depthgtvls = tensor2disp(1 / data_blob['depthmap'], vmax=0.15, viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(depthgtvls)], axis=1) img_val = np.concatenate( [np.array(img_val_up), np.array(img_val_mid2)], axis=0) self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], data_blob['flowmap'], data_blob['insmap'], outputs) self.writer.add_image('{}_X'.format(tagname), (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def write_vls(self, data_blob, outputs, flowselector, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() figmask_flow = tensor2disp(flowselector, vmax=1, viewind=0) insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs[('depth', 2)], vmax=0.15, viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(figmask_flow)], axis=1) img_val = np.concatenate( [np.array(img_val_up), np.array(img_val_mid2)], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], data_blob['flowgt_vls'], data_blob['insmap'], outputs) self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def write_vls(self, data_blob, outputs, flowselector, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() figmask_flow = tensor2disp(flowselector, vmax=1, viewind=0) insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs[('depth', 2)], vmax=0.15, viewind=0) flowvls = flow_to_image( outputs[('flowpred', 2)][0].detach().cpu().permute([1, 2, 0]).numpy(), rad_max=10) imgrecon = tensor2rgb(outputs[('reconImg', 2)], viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(figmask_flow)], axis=1) img_val_mid3 = np.concatenate( [np.array(imgrecon), np.array(flowvls)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid2), np.array(img_val_mid3) ], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step)
def write_vls_eval(self, data_blob, outputs, tagname, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) flowvls = flow_to_image(outputs[-1][0].detach().cpu().permute([1, 2, 0]).numpy(), rad_max=50) img_val_up = np.concatenate([np.array(insvls), np.array(flowvls)], axis=1) self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val_up).float() / 255).permute([2, 0, 1]), step)
def write_vls_eval(self, data_blob, outputs, tagname, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'], vmax=0.15, viewind=0) imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid = np.concatenate([np.array(depthpredvls), np.array(imgrecon)], axis=1) img_val = np.concatenate([np.array(img_val_up), np.array(img_val_mid)], axis=0) self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step)
def write_vls(self, data_blob, outputs, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'], vmax=0.15, viewind=0) imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0) flow_RAFT = Image.fromarray( flow_to_image(data_blob['flowpred'][0].permute([1, 2, 0]).cpu().numpy())) flow_pred = Image.fromarray( flow_to_image(outputs[('flowpred', 2)][0].permute([1, 2, 0]).detach().cpu().numpy())) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid = np.concatenate( [np.array(depthpredvls), np.array(imgrecon)], axis=1) img_val_flow = np.concatenate( [np.array(flow_RAFT), np.array(flow_pred)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid), np.array(img_val_flow) ], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], outputs) self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step) X = self.vls_objmvment(np.array(insvls), data_blob['insmap'], data_blob['posepred']) self.writer.add_image('objmvment', (torch.from_numpy(X).float() / 255).permute([2, 0, 1]), step)
def write_vls(self, data_blob, outputs, depthselector, step): img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8) img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8) insmap = data_blob['insmap'][0].squeeze().numpy() figmask_depth = tensor2disp(depthselector, vmax=1, viewind=0) insvls = vls_ins(img1, insmap) depthpredvls = tensor2disp(1 / outputs['depth_predictions'][-1], vmax=0.15, viewind=0) flowvls = flow_to_image(outputs['flowpred'][0].detach().cpu().permute( [1, 2, 0]).numpy(), rad_max=50) imgrecon = tensor2rgb(outputs['img1_recon'], viewind=0) img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1) img_val_mid2 = np.concatenate( [np.array(depthpredvls), np.array(figmask_depth)], axis=1) img_val_mid3 = np.concatenate( [np.array(imgrecon), np.array(flowvls)], axis=1) img_val = np.concatenate([ np.array(img_val_up), np.array(img_val_mid2), np.array(img_val_mid3) ], axis=0) self.writer.add_image('predvls', (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step) X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'], data_blob['flowmap'], data_blob['insmap'], outputs) self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute( [2, 0, 1]), step)
def validate_RANSAC_odom_relpose(args, eval_loader, banins=False, bangrad=False, samplenum=50000): if bangrad: gradComputer = None else: gradComputer = GradComputer() for val_id, data_blob in enumerate(tqdm(eval_loader)): insmap = data_blob['insmap'] intrinsic = data_blob['intrinsic'] flowpred = data_blob['flowpred_RAFT'] mdDepth_pred = data_blob['mdDepth'] pose_bs = data_blob['posepred_bs'] tag = data_blob['tag'][0] if torch.sum(torch.abs(data_blob['img1'] - data_blob['img2'])) < 1: R = np.eye(3) t = np.array([[0, 0, -1]]).T scale = 0 else: R, t, scale, _ = inf_pose_flow(flowpred, insmap, mdDepth_pred, intrinsic, int(val_id + 10), gradComputer=gradComputer, banins=banins, samplenum=samplenum) self_pose = np.eye(4) self_pose[0:3, 0:3] = R self_pose[0:3, 3:4] = t * scale pose_bs_np = pose_bs[0].cpu().numpy() pose_bs_np = pose_bs_np @ np.linalg.inv(pose_bs_np[0]) @ self_pose pose_bs_np[0] = self_pose seq, frmidx = tag.split(' ') exportfold = os.path.join(args.vls_root, seq, 'image_02') insmap_np = data_blob['insmap'].squeeze().numpy() xx, yy = np.meshgrid(range(insmap_np.shape[1]), range(insmap_np.shape[0]), indexing='xy') fig, ax = plt.subplots(figsize=(16,9)) ax.imshow(vls_ins(np.array(tensor2rgb(data_blob['img1'], viewind=0)), insmap_np)) for k in np.unique(insmap_np): if k == 0: continue xxf = xx[insmap_np == k] yyf = yy[insmap_np == k] xmin = xxf.min() xmax = xxf.max() ymin = yyf.min() ymax = yyf.max() if (ymax - ymin) * (xmax - xmin) < 1000: continue rect = patches.Rectangle((xmin, ymax), xmax - xmin, ymin - ymax, linewidth=1, facecolor='none', edgecolor='r') ax.add_patch(rect) ins_relpose = pose_bs[0, k].cpu().numpy() @ np.linalg.inv(pose_bs[0, 0].cpu().numpy()) mvdist = np.sqrt(np.sum(ins_relpose[0:3, 3:4] ** 2)) mvdist_transed = np.sqrt(np.sum((pose_bs_np[k] @ np.linalg.inv(pose_bs_np[0]))[0:3, 3:4] ** 2)) assert np.abs(mvdist_transed - mvdist) < 1e-3 ax.text(xmin + 5, ymin + 10, '%.3f' % mvdist, fontsize=6, c='r', weight='bold') plt.axis('off') os.makedirs(exportfold, exist_ok=True) plt.savefig(os.path.join(exportfold, "{}.png".format(frmidx)), bbox_inches='tight', pad_inches=0) plt.close()