def __getitem__(self, index): if not self.init_seed: worker_info = torch.utils.data.get_worker_info() if worker_info is not None: torch.manual_seed(worker_info.id) np.random.seed(worker_info.id) random.seed(worker_info.id) self.init_seed = True index = index % len(self.image_list) flow = frame_utils.read_gen(self.flow_list[index]) img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8)[..., :3] img2 = np.array(img2).astype(np.uint8)[..., :3] flow = np.array(flow).astype(np.float32) if self.do_augument: img1, img2, flow = self.augumentor(img1, img2, flow) img1 = torch.from_numpy(img1).permute(2, 0, 1).float() img2 = torch.from_numpy(img2).permute(2, 0, 1).float() flow = torch.from_numpy(flow).permute(2, 0, 1).float() valid = torch.ones_like(flow[0]) return img1, img2, flow, valid
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) intrinsic = copy.deepcopy(self.intrinsic_list[index]) RANSACPose = copy.deepcopy(self.RANSACPose_rec[index]) IMUPose = copy.deepcopy(self.IMUPose_rec[index]) if self.ins_root is not None: inspred = np.array(Image.open(self.inspred_list[index])).astype( np.int) else: inspred = None mdDepth = np.array(Image.open(self.mdPred_list[index])).astype( np.float32) / 256.0 stereogt = np.array(Image.open(self.stereoGt_list[index])).astype( np.float32) / 256.0 flowpred_RAFT, _ = readFlowKITTI(self.flowPred_list[index]) flowgt, flowgt_valid = readFlowKITTI(self.flowGt_list[index]) data_blob = self.wrapup(img1=img1, img2=img2, intrinsic=intrinsic, insmap=inspred, mdDepth=mdDepth, stereogt=stereogt, flowpred_RAFT=flowpred_RAFT, flowgt=flowgt, flowgt_valid=flowgt_valid, RANSACPose=RANSACPose, IMUPose=IMUPose, tag=self.entries_map[index]) return data_blob
def read_imgpair_flow_depth_instance_intrinsic(self, index): img1 = np.array(read_gen(self.rgb1_paths[index])).astype(np.uint8) img2 = np.array(read_gen(self.rgb2_paths[index])).astype(np.uint8) flowmap = np.array(readFlowVRKitti(self.flow_paths[index])).astype( np.float32) depthmap = np.array( cv2.imread(self.depth_paths[index], cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH)).astype(np.float32) depthmap = depthmap / 100 instancemap = np.array(Image.open(self.ins_paths[index])) instancemap = np.array(instancemap).astype(np.int32) - 1 sceneidx, envn, frmidx = self.entries[index].split(' ') frmidx = int(frmidx) camidx = int(0) scene_name = "Scene{}_{}".format(sceneidx.zfill(2), envn) intrinsic_key = "{}_frm{}_cam{}".format(scene_name, str(frmidx).zfill(5), camidx) intrinsic = copy.deepcopy(self.intrinsic_dict[intrinsic_key]) return img1, img2, flowmap, depthmap, instancemap, intrinsic
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) intrinsic = copy.deepcopy(self.intrinsic_list[index]) if self.ins_root is not None: inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int) if self.banins: inspred = inspred * 0 inspred = inspred.astype(np.int) else: inspred = None if len(self.bsposepath_list[index]) == 0: posepred_bs = np.eye(4) posepred_bs = np.expand_dims(posepred_bs, axis=0) posepred_bs = np.repeat(posepred_bs, axis=0, repeats=int(inspred.max() + 1)) else: posepred_bs = pickle.load(open(self.bsposepath_list[index], "rb")) mdDepth = np.array(Image.open(self.mdPred_list[index])).astype(np.float32) / 256.0 flowpred_RAFT, valid_flow = readFlowKITTI(self.flowPred_list[index]) data_blob = self.wrapup(img1=img1, img2=img2, intrinsic=intrinsic, insmap=inspred, mdDepth=mdDepth, flowpred_RAFT=flowpred_RAFT, posepred_bs=posepred_bs, tag=self.entries[index]) return data_blob
def __getitem__(self, index): if self.is_test: img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8)[..., :3] img2 = np.array(img2).astype(np.uint8)[..., :3] img1 = torch.from_numpy(img1).permute(2, 0, 1).float() img2 = torch.from_numpy(img2).permute(2, 0, 1).float() return img1, img2, self.extra_info[index] if not self.init_seed: worker_info = torch.utils.data.get_worker_info() if worker_info is not None: torch.manual_seed(worker_info.id) np.random.seed(worker_info.id) random.seed(worker_info.id) self.init_seed = True index = index % len(self.image_list) valid = None if self.sparse: flow, valid = frame_utils.readFlowKITTI(self.flow_list[index]) else: flow = frame_utils.read_gen(self.flow_list[index]) img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) flow = np.array(flow).astype(np.float32) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) # grayscale images if len(img1.shape) == 2: img1 = np.tile(img1[..., None], (1, 1, 3)) img2 = np.tile(img2[..., None], (1, 1, 3)) else: img1 = img1[..., :3] img2 = img2[..., :3] if self.augmentor is not None: if self.sparse: img1, img2, flow, valid = self.augmentor( img1, img2, flow, valid) else: img1, img2, flow = self.augmentor(img1, img2, flow) img1 = torch.from_numpy(img1).permute(2, 0, 1).float() img2 = torch.from_numpy(img2).permute(2, 0, 1).float() flow = torch.from_numpy(flow).permute(2, 0, 1).float() if valid is not None: valid = torch.from_numpy(valid) else: valid = (flow[0].abs() < 1000) & (flow[1].abs() < 1000) return img1, img2, flow, valid.float()
def validate_kitti_colorjitter(gpu, model, args, ngpus_per_node, eval_entries, iters=24): """ Peform validation using the KITTI-2015 (train) split """ interval = np.floor(len(eval_entries) / ngpus_per_node).astype(np.int).item() if gpu == ngpus_per_node - 1: stidx = int(interval * gpu) edidx = len(eval_entries) else: stidx = int(interval * gpu) edidx = int(interval * (gpu + 1)) print("Initialize Instance on Gpu %d, from %d to %d, total %d" % (gpu, stidx, edidx, len(eval_entries))) from tqdm import tqdm model.eval() model.cuda(gpu) with torch.no_grad(): for val_id, entry in enumerate(tqdm(remove_dup(eval_entries[stidx : edidx]))): seq, index = entry.split(' ') index = int(index) if os.path.exists(os.path.join(args.dataset, seq, 'image_02', 'data', "{}.png".format(str(index).zfill(10)))): tmproot = args.dataset else: tmproot = args.odom_root img1path = os.path.join(tmproot, seq, 'image_02', 'data', "{}.png".format(str(index).zfill(10))) img2path = os.path.join(tmproot, seq, 'image_02', 'data', "{}.png".format(str(index + 1).zfill(10))) if not os.path.exists(img2path): img2path = img1path image1 = frame_utils.read_gen(img1path) image2 = frame_utils.read_gen(img2path) image1 = np.array(image1).astype(np.uint8) image2 = np.array(image2).astype(np.uint8) image1 = torch.from_numpy(image1).permute([2, 0, 1]).float() image2 = torch.from_numpy(image2).permute([2, 0, 1]).float() svfold = os.path.join(args.exportroot, seq, 'image_02') svpath = os.path.join(args.exportroot, seq, 'image_02', "{}.png".format(str(index).zfill(10))) os.makedirs(svfold, exist_ok=True) image1 = image1[None].cuda(gpu) image2 = image2[None].cuda(gpu) padder = InputPadder(image1.shape, mode='kitti') image1, image2 = padder.pad(image1, image2) flow_low, flow_pr = model(image1, image2, iters=iters, test_mode=True) flow = padder.unpad(flow_pr[0]).cpu() frame_utils.writeFlowKITTI(svpath, flow.permute(1, 2, 0).numpy()) # Image.fromarray(flow_viz.flow_to_image(flow.permute(1, 2, 0).numpy())).show() # tensor2rgb(image1 / 255.0, viewind=0).show() return
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) sequence = self.image_list[index][2] frame = self.image_list[index][3] img1 = np.array(img1).astype(np.uint8)[..., :3] img2 = np.array(img2).astype(np.uint8)[..., :3] img1 = torch.from_numpy(img1).permute(2, 0, 1).float() img2 = torch.from_numpy(img2).permute(2, 0, 1).float() return img1, img2, sequence, frame
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) intrinsic = copy.deepcopy(self.intrinsic_list[index]) if self.ins_root is not None: inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int) else: inspred = None mdDepth = np.array(Image.open(self.mdPred_list[index])).astype(np.float32) / 256.0 flowpred_RAFT, valid_flow = readFlowKITTI(self.flowPred_list[index]) data_blob = self.wrapup(img1=img1, img2=img2, intrinsic=intrinsic, insmap=inspred, mdDepth=mdDepth, flowpred_RAFT=flowpred_RAFT, tag=self.entries[index]) return data_blob
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) intrinsic = copy.deepcopy(self.intrinsic) mdDepth = cv2.imread(self.mdPred_list[index], -1) mdDepth = mdDepth.astype(np.float32) / 1000.0 gtDepth = cv2.imread(self.depthgt_list[index], -1) gtDepth = gtDepth.astype(np.float32) / 1000.0 flowpred_RAFT, valid_flow = readFlowKITTI(self.flowPred_list[index]) data_blob = self.wrapup(img1=img1, img2=img2, intrinsic=intrinsic, mdDepth=mdDepth, gtDepth=gtDepth, flowpred_RAFT=flowpred_RAFT, tag=self.entries[index]) return data_blob
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) depth = np.array(Image.open(self.depth_list[index])).astype( np.float32) / 256.0 depthvls = np.array(Image.open(self.depthvls_list[index])).astype( np.float32) / 256.0 intrinsic = copy.deepcopy(self.intrinsic_list[index]) rel_pose = copy.deepcopy(self.pose_list[index]) inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int) posepred = pickle.load(open(self.RANSACPose_list[index], "rb")) inspred, posepred = self.pad_clip_ins(insmap=inspred, posepred=posepred) if self.mdPred_root is not None: mdDepth_pred = np.array(Image.open( self.mdDepthpath_list[index])).astype(np.float32) / 256.0 else: mdDepth_pred = None img1, img2, depth, depthvls, mdDepth_pred, inspred, intrinsic = self.aug_crop( img1, img2, depth, depthvls, mdDepth_pred, inspred, intrinsic) if self.istrain and not self.muteaug: img1, img2 = self.colorjitter(img1, img2) data_blob = self.wrapup(img1=img1, img2=img2, depthmap=depth, depthvls=depthvls, mdDepth_pred=mdDepth_pred, intrinsic=intrinsic, insmap=inspred, rel_pose=rel_pose, posepred=posepred, tag=self.entries[index]) return data_blob
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) depth = cv2.imread(self.depth_list[index], -1) / 1000.0 intrinsic = copy.deepcopy(self.intrinsic) if self.istrain: np.random.seed(index + int(time.time())) samp_rep = int(np.random.choice(list(range(self.nposs)), 1).item()) else: samp_rep = int(0) mdDepth_pred = cv2.imread( self.mdDepthpath_list[index].format(str(samp_rep).zfill(3)), -1) / 1000.0 posepred = pickle.load( open(self.RANSACPose_list[index].format(str(samp_rep).zfill(3)), "rb")) flowpred, _ = frame_utils.readFlowKITTI(self.flowpred_list[index]) img1, img2, depth, mdDepth_pred, flowpred, intrinsic = self.aug_crop( img1, img2, depth, mdDepth_pred, flowpred, intrinsic) if self.istrain: img1, img2 = self.colorjitter(img1, img2) data_blob = self.wrapup(img1=img1, img2=img2, depthmap=depth, mdDepth_pred=mdDepth_pred, flowpred=flowpred, intrinsic=intrinsic, posepred=posepred, tag=self.entries[index]) return data_blob
def validate_kitti_colorjitter(gpu, model, args, ngpus_per_node, eval_entries, iters=24): """ Peform validation using the KITTI-2015 (train) split """ interval = np.floor(len(eval_entries) / ngpus_per_node).astype( np.int).item() if gpu == ngpus_per_node - 1: stidx = int(interval * gpu) edidx = len(eval_entries) else: stidx = int(interval * gpu) edidx = int(interval * (gpu + 1)) print("Initialize Instance on Gpu %d, from %d to %d, total %d" % (gpu, stidx, edidx, len(eval_entries))) from tqdm import tqdm model.eval() model.cuda(gpu) with torch.no_grad(): for val_id, entry in enumerate( tqdm(remove_dup(eval_entries[stidx:edidx]))): seq, index = entry.split(' ') index = int(index) img1path = os.path.join(args.dataset, seq, 'rgb_{}.png'.format(str(index).zfill(5))) img2path = os.path.join( args.dataset, seq, 'rgb_{}.png'.format(str(index + 1).zfill(5))) if not os.path.exists(img2path) and not os.path.exists( img2path.replace('.png', '.jpg')): img2path = img1path if not os.path.exists(img1path): img1path = img1path.replace('.png', '.jpg') img2path = img2path.replace('.png', '.jpg') image1 = frame_utils.read_gen(img1path) image2 = frame_utils.read_gen(img2path) image1 = np.array(image1).astype(np.uint8) image2 = np.array(image2).astype(np.uint8) image1 = torch.from_numpy(image1).permute([2, 0, 1]).float() image2 = torch.from_numpy(image2).permute([2, 0, 1]).float() svfold = os.path.join( args.exportroot, seq, ) svpath = os.path.join(args.exportroot, seq, '{}.png'.format(str(index).zfill(5))) os.makedirs(svfold, exist_ok=True) image1 = image1[None].cuda(gpu) image2 = image2[None].cuda(gpu) flow_low, flow_pr = model(image1, image2, iters=iters, test_mode=True) flow = flow_pr.squeeze(0) flow_numpy = flow.cpu().permute(1, 2, 0).numpy() frame_utils.writeFlowKITTI(svpath, flow_numpy) if args.dovls: vlsflow = Image.fromarray(flow_viz.flow_to_image(flow_numpy)) vlsrgb1 = tensor2rgb(image1 / 255.0, viewind=0) vlsrgb2 = tensor2rgb(image2 / 255.0, viewind=0) w, h = vlsrgb2.size xx, yy = np.meshgrid(range(w), range(h), indexing='xy') xxf = xx.flatten() yyf = yy.flatten() rndidx = np.random.randint(0, xxf.shape[0], 100) xxf_rnd = xxf[rndidx] yyf_rnd = yyf[rndidx] flowx = flow_numpy[yyf_rnd, xxf_rnd, 0] flowy = flow_numpy[yyf_rnd, xxf_rnd, 1] fig = plt.figure(figsize=(12, 9)) plt.subplot(2, 1, 1) plt.scatter(xxf_rnd, yyf_rnd, 1, 'r') plt.imshow(vlsrgb1) plt.subplot(2, 1, 2) plt.scatter(flowx + xxf_rnd, flowy + yyf_rnd, 1, 'r') plt.imshow(vlsrgb2) plt.savefig( os.path.join( "/media/shengjie/disk1/Prediction/nyuv2_flow_vls", "{}_{}.jpg".format(seq, str(index).zfill(5)))) plt.close() # combined_left = np.concatenate([np.array(vlsrgb1), np.array(vlsrgb2)], axis=0) # combined_right = np.concatenate([np.array(vlsflow), np.array(vlsflow)], axis=0) # combined = np.concatenate([combined_left, combined_right], axis=1) # vls_sv_root = os.makedirs("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", exist_ok=True) # Image.fromarray(combined).save(os.path.join("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", "{}_{}.jpg".format(seq, str(index).zfill(5)))) return
def __getitem__(self, index): if self.is_test: frame_id = self.image_list[index][0] frame_id = frame_id.split('/')[-1] img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8)[..., :3] img2 = np.array(img2).astype(np.uint8)[..., :3] img1 = torch.from_numpy(img1).permute(2, 0, 1).float() img2 = torch.from_numpy(img2).permute(2, 0, 1).float() return img1, img2, frame_id else: if not self.init_seed: worker_info = torch.utils.data.get_worker_info() if worker_info is not None: np.random.seed(worker_info.id) random.seed(worker_info.id) self.init_seed = True index = index % len(self.image_list) frame_id = self.image_list[index][0] frame_id = frame_id.split('/')[-1] img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) flow, valid = frame_utils.readFlowKITTI(self.flow_list[index]) img1 = np.array(img1).astype(np.uint8)[..., :3] img2 = np.array(img2).astype(np.uint8)[..., :3] if self.do_augument: img1, img2, flow, valid = self.augumentor( img1, img2, flow, valid) img1 = torch.from_numpy(img1).permute(2, 0, 1).float() img2 = torch.from_numpy(img2).permute(2, 0, 1).float() flow = torch.from_numpy(flow).permute(2, 0, 1).float() valid = torch.from_numpy(valid).float() if self.do_pad: ht, wd = img1.shape[1:] pad_ht = (((ht // 8) + 1) * 8 - ht) % 8 pad_wd = (((wd // 8) + 1) * 8 - wd) % 8 pad_ht1 = [0, pad_ht] pad_wd1 = [pad_wd // 2, pad_wd - pad_wd // 2] pad = pad_wd1 + pad_ht1 img1 = img1.view(1, 3, ht, wd) img2 = img2.view(1, 3, ht, wd) flow = flow.view(1, 2, ht, wd) valid = valid.view(1, 1, ht, wd) img1 = torch.nn.functional.pad(img1, pad, mode='replicate') img2 = torch.nn.functional.pad(img2, pad, mode='replicate') flow = torch.nn.functional.pad(flow, pad, mode='constant', value=0) valid = torch.nn.functional.pad(valid, pad, mode='replicate', value=0) img1 = img1.view(3, ht + pad_ht, wd + pad_wd) img2 = img2.view(3, ht + pad_ht, wd + pad_wd) flow = flow.view(2, ht + pad_ht, wd + pad_wd) valid = valid.view(ht + pad_ht, wd + pad_wd) if self.is_test: return img1, img2, flow, valid, frame_id return img1, img2, flow, valid
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) depthvls = np.array(Image.open(self.depthvls_list[index])).astype(np.float32) / 256.0 intrinsic = copy.deepcopy(self.intrinsic_list[index]) rel_pose = copy.deepcopy(self.pose_list[index]) inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int) if self.RANSACPose_root is not None: np.random.seed(index + int(time.time())) rep_idces = np.random.choice(list(range(self.npossibility)), self.num_samples, replace=False) # rep_idces = np.array([0, 1, 2, 3]) # rep_idces = np.array([0]) posepred = list() ang_decps_l = list() scl_decps_l = list() mvd_decps_l = list() for rep_idx in rep_idces: posepred_cur = pickle.load(open(self.RANSACPose_list[index].format(str(rep_idx).zfill(3)), "rb")) for l in self.linlogdedge: scaledpose = copy.deepcopy(posepred_cur) for kk in range(scaledpose.shape[0]): tmpscale = np.sqrt(np.sum(scaledpose[kk, 0:3, 3] ** 2) + 1e-12) adjustedscale = np.exp(np.log(tmpscale) + l) scaledpose[kk, 0:3, 3] = scaledpose[kk, 0:3, 3] / tmpscale * adjustedscale posepred.append(scaledpose) posepred_decps = copy.deepcopy(posepred_cur) ang_decps = np.zeros([posepred_decps.shape[0], 3]) scl_decps = np.zeros([posepred_decps.shape[0], 1]) mvd_decps = np.zeros([posepred_decps.shape[0], 3]) for k in range(posepred_decps.shape[0]): ang_decps[k, :] = self.rot2ang(posepred_decps[k]) tmps = np.sum(posepred_decps[k, 0:3, 3] ** 2) if tmps == 0: scl_decps[k, 0] = 1e-6 mvd_decps[k, :] = np.array([0, 0, -1]) else: scl_decps[k, 0] = np.sqrt(tmps) mvd_decps[k, :] = posepred_decps[k, 0:3, 3] / scl_decps[k, 0] ang_decps_l.append(ang_decps) scl_decps_l.append(scl_decps) mvd_decps_l.append(mvd_decps) posepred = np.stack(posepred, axis=0) ang_decps = np.stack(ang_decps_l, axis=0) scl_decps = np.stack(scl_decps_l, axis=0) mvd_decps = np.stack(mvd_decps_l, axis=0) else: posepred = None ang_decps = None scl_decps = None mvd_decps = None inspred, posepred, ang_decps_pad, scl_decps_pad, mvd_decps_pad = self.pad_clip_ins(insmap=inspred, posepred=posepred, ang_decps=ang_decps, scl_decps=scl_decps, mvd_decps=mvd_decps) if not hasattr(self, 'deepv2dpred_root'): self.deepv2dpred_root = None if self.depth_root is not None: depth = np.array(Image.open(self.depth_list[index])).astype(np.float32) / 256.0 else: depth = None if self.deepv2dpred_root is not None: depthpred_deepv2d = np.array(Image.open(self.deepv2dpredpath_list[index])).astype(np.float32) / 256.0 posepred_deepv2d = read_deepv2d_pose(self.deepv2dpredpose_list[index]) else: depthpred_deepv2d = None posepred_deepv2d = None if self.mdPred_root is not None: mdDepth_pred = np.array(Image.open(self.mdDepthpath_list[index])).astype(np.float32) / 256.0 else: mdDepth_pred = None if self.flowPred_root is not None: flowpred_RAFT, valid_flow = readFlowKITTI(self.flowpredpath_list[index]) else: flowpred_RAFT = None img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic = self.aug_crop(img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic) if self.depth_root is not None: flowgt = self.get_gt_flow(depth=depth, valid=(inspred==0) * (depth>0), intrinsic=intrinsic, rel_pose=rel_pose) else: flowgt = None flowgt_vls = self.get_gt_flow(depth=depthvls, valid=(inspred==0) * (depthvls>0), intrinsic=intrinsic, rel_pose=rel_pose) if self.istrain and not self.muteaug: img1, img2 = self.colorjitter(img1, img2) entry = self.entries[index] seq, frmidx = entry.split(' ') frmidx = int(frmidx) seq = seq.split('/')[1] IMUposes_seg = list() RANSACposes_seg = list() IMU_poses_reced = self.IMU_poses_rec[seq] RANSAC_poses_reced = self.RANSAC_poses_rec[seq] for k in range(frmidx - self.span, frmidx + self.span): if k < 0: IMUposes_seg.append(np.eye(4)) RANSACposes_seg.append(np.eye(4)) elif k >= len(IMU_poses_reced): IMUposes_seg.append(np.eye(4)) RANSACposes_seg.append(np.eye(4)) else: IMUposes_seg.append(IMU_poses_reced[k]) RANSACposes_seg.append(RANSAC_poses_reced[k]) # Forward move IMUlocations1 = list() leftarrs1 = list() rightarrs1 = list() accumM = np.eye(4) leftarr = np.eye(4) rightarr = np.eye(4) for k in range(len(IMUposes_seg)): accumM = IMUposes_seg[k] @ accumM if k < self.span: rightarr = RANSACposes_seg[k] @ rightarr if k > self.span: leftarr = RANSACposes_seg[k] @ leftarr if k >= self.span: IMUlocations1.append(np.linalg.inv(accumM)[0:3, 3:4]) leftarrs1.append(leftarr) rightarrs1.append(rightarr) # print(np.sum(np.abs(np.linalg.inv(leftarr @ rel_pose @ rightarr)[0:3, 3] - IMUlocations1[k-self.span]))) IMUlocations2 = list() leftarrs2 = list() rightarrs2 = list() accumM = np.eye(4) leftarr = np.eye(4) rightarr = np.eye(4) for k in range(len(IMUposes_seg)): accumM = np.linalg.inv(IMUposes_seg[-k]) @ accumM if k < self.span: rightarr = np.linalg.inv(RANSACposes_seg[-k]) @ rightarr if k > self.span: leftarr = np.linalg.inv(RANSACposes_seg[-k]) @ leftarr if k >= self.span: IMUlocations2.append(np.linalg.inv(accumM)[0:3, 3:4]) leftarrs2.append(leftarr) rightarrs2.append(rightarr) # print(np.sum(np.abs(np.linalg.inv(leftarr @ np.linalg.inv(rel_pose) @ rightarr)[0:3, 3] - IMUlocations2[k-self.span]))) IMUlocations1 = np.stack(IMUlocations1) leftarrs1 = np.stack(leftarrs1) rightarrs1 = np.stack(rightarrs1) IMUlocations2 = np.stack(IMUlocations2) leftarrs2 = np.stack(leftarrs2) rightarrs2 = np.stack(rightarrs2) data_blob = self.wrapup(img1=img1, img2=img2, flowgt=flowgt, flowgt_vls=flowgt_vls, depthmap=depth, depthvls=depthvls, depthpred_deepv2d=depthpred_deepv2d, mdDepth_pred=mdDepth_pred, intrinsic=intrinsic, insmap=inspred, flowpred=flowpred_RAFT, rel_pose=rel_pose, posepred=posepred, posepred_deepv2d=posepred_deepv2d, ang_decps_pad=ang_decps_pad, scl_decps_pad=scl_decps_pad, mvd_decps_pad=mvd_decps_pad, tag=self.entries[index], IMUlocations1=IMUlocations1, leftarrs1=leftarrs1, rightarrs1=rightarrs1, IMUlocations2=IMUlocations2, leftarrs2=leftarrs2, rightarrs2=rightarrs2) return data_blob
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) depthvls = np.array(Image.open(self.depthvls_list[index])).astype( np.float32) / 256.0 intrinsic = copy.deepcopy(self.intrinsic_list[index]) rel_pose = copy.deepcopy(self.pose_list[index]) inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int) if self.RANSACPose_root is not None: np.random.seed(index + int(time.time())) if self.num_samples == 1: rep_idces = [0] elif self.num_samples == 2: rep_idces = [0, 1] else: rep_idces = [0, 1, 2, 3] # rep_idces = [0, 0, 0, 0] posepred = list() ang_decps_l = list() scl_decps_l = list() mvd_decps_l = list() for rep_idx in rep_idces: posepred_cur = pickle.load( open( self.RANSACPose_list[index].format( str(rep_idx).zfill(3)), "rb")) for l in self.linlogdedge: scaledpose = copy.deepcopy(posepred_cur) for kk in range(scaledpose.shape[0]): tmpscale = np.sqrt( np.sum(scaledpose[kk, 0:3, 3]**2) + 1e-12) adjustedscale = np.exp(np.log(tmpscale) + l) scaledpose[kk, 0:3, 3] = scaledpose[ kk, 0:3, 3] / tmpscale * adjustedscale posepred.append(scaledpose) posepred_decps = copy.deepcopy(posepred_cur) ang_decps = np.zeros([posepred_decps.shape[0], 3]) scl_decps = np.zeros([posepred_decps.shape[0], 1]) mvd_decps = np.zeros([posepred_decps.shape[0], 3]) for k in range(posepred_decps.shape[0]): ang_decps[k, :] = self.rot2ang(posepred_decps[k]) tmps = np.sum(posepred_decps[k, 0:3, 3]**2) if tmps == 0: scl_decps[k, 0] = 1e-6 mvd_decps[k, :] = np.array([0, 0, -1]) else: scl_decps[k, 0] = np.sqrt(tmps) mvd_decps[k, :] = posepred_decps[k, 0:3, 3] / scl_decps[k, 0] ang_decps_l.append(ang_decps) scl_decps_l.append(scl_decps) mvd_decps_l.append(mvd_decps) posepred = np.stack(posepred, axis=0) ang_decps = np.stack(ang_decps_l, axis=0) scl_decps = np.stack(scl_decps_l, axis=0) mvd_decps = np.stack(mvd_decps_l, axis=0) else: posepred = None ang_decps = None scl_decps = None mvd_decps = None relpose_odom_gt = copy.deepcopy(self.gt_relpose_rec[index]) inspred, posepred, ang_decps_pad, scl_decps_pad, mvd_decps_pad = self.pad_clip_ins( insmap=inspred, posepred=posepred, ang_decps=ang_decps, scl_decps=scl_decps, mvd_decps=mvd_decps) if not hasattr(self, 'deepv2dpred_root'): self.deepv2dpred_root = None if self.depth_root is not None: depth = np.array(Image.open(self.depth_list[index])).astype( np.float32) / 256.0 else: depth = None if self.deepv2dpred_root is not None: depthpred_deepv2d = np.array( Image.open(self.deepv2dpredpath_list[index])).astype( np.float32) / 256.0 posepred_deepv2d = read_deepv2d_pose( self.deepv2dpredpose_list[index]) else: depthpred_deepv2d = None posepred_deepv2d = None if self.mdPred_root is not None: mdDepth_pred = np.array(Image.open( self.mdDepthpath_list[index])).astype(np.float32) / 256.0 else: mdDepth_pred = None if self.flowPred_root is not None: flowpred_RAFT, valid_flow = readFlowKITTI( self.flowpredpath_list[index]) else: flowpred_RAFT = None img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic = self.aug_crop( img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic) if self.depth_root is not None: flowgt = self.get_gt_flow(depth=depth, valid=(inspred == 0) * (depth > 0), intrinsic=intrinsic, rel_pose=rel_pose) else: flowgt = None flowgt_vls = self.get_gt_flow(depth=depthvls, valid=(inspred == 0) * (depthvls > 0), intrinsic=intrinsic, rel_pose=rel_pose) if self.istrain and not self.muteaug: img1, img2 = self.colorjitter(img1, img2) data_blob = self.wrapup(img1=img1, img2=img2, flowgt=flowgt, flowgt_vls=flowgt_vls, depthmap=depth, depthvls=depthvls, depthpred_deepv2d=depthpred_deepv2d, mdDepth_pred=mdDepth_pred, intrinsic=intrinsic, insmap=inspred, flowpred=flowpred_RAFT, rel_pose=rel_pose, posepred=posepred, posepred_deepv2d=posepred_deepv2d, relpose_odom_gt=relpose_odom_gt, ang_decps_pad=ang_decps_pad, scl_decps_pad=scl_decps_pad, mvd_decps_pad=mvd_decps_pad, tag=self.entries[index]) return data_blob
def validate_kitti_colorjitter(gpu, model, args, ngpus_per_node, eval_entries, iters=24): """ Peform validation using the KITTI-2015 (train) split """ stidx = 0 edidx = len(eval_entries) print("Initialize Instance on Gpu %d, from %d to %d, total %d" % (gpu, stidx, edidx, len(eval_entries))) from tqdm import tqdm model.eval() model.cuda(gpu) totnum = 0 dr = 0 with torch.no_grad(): for val_id, entry in enumerate(remove_dup(eval_entries[stidx:edidx])): seq, index = entry.split(' ') index = int(index) if os.path.exists( os.path.join(args.dataset, seq, 'image_02', 'data', "{}.png".format(str(index).zfill(10)))): tmproot = args.dataset else: tmproot = args.odom_root img1path = os.path.join(tmproot, seq, 'image_02', 'data', "{}.png".format(str(index).zfill(10))) img2path = os.path.join(tmproot, seq, 'image_02', 'data', "{}.png".format(str(index + 1).zfill(10))) if not os.path.exists(img2path): img2path = img1path image1 = frame_utils.read_gen(img1path) image2 = frame_utils.read_gen(img2path) image1 = np.array(image1).astype(np.uint8) image2 = np.array(image2).astype(np.uint8) h, w, _ = image1.shape crph = h - args.evalheight image1 = image1[crph:h, 0:args.evalwidth, :] image2 = image2[crph:h, 0:args.evalwidth, :] image1 = torch.from_numpy(image1).permute([2, 0, 1]).float() image2 = torch.from_numpy(image2).permute([2, 0, 1]).float() image1 = image1[None].cuda(gpu) image2 = image2[None].cuda(gpu) # padder = InputPadder(image1.shape, mode='kitti') # image1, image2 = padder.pad(image1, image2) st = time.time() flow_low, flow_pr = model(image1, image2, iters=24, test_mode=True) dr += time.time() - st totnum += 1 print("%d Samples, Ave sec/frame: %f, Mem: %f Gb" % (totnum, dr / totnum, float(torch.cuda.memory_allocated() / 1024 / 1024 / 1024))) return
def __getitem__(self, index): img1 = frame_utils.read_gen(self.image_list[index][0]) img2 = frame_utils.read_gen(self.image_list[index][1]) img1 = np.array(img1).astype(np.uint8) img2 = np.array(img2).astype(np.uint8) h, w, _ = img1.shape size_str = "{}_{}".format(str(h), str(w)) if not self.bandepth: depth = np.array(Image.open(self.depth_list[index])).astype( np.float32) / 256.0 depthvls = np.array(Image.open(self.depthvls_list[index])).astype( np.float32) / 256.0 else: depth = np.zeros([h, w]) depthvls = np.zeros([h, w]) intrinsic = copy.deepcopy(self.intrinsic_list[index]) rel_pose = copy.deepcopy(self.pose_list[index]) inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int) if self.prediction_root is not None: depthpred = np.array(Image.open( self.predDepthpath_list[index])).astype(np.float32) / 256.0 else: depthpred = None assert not (self.prediction_root is not None and self.RANSACPose_root is not None) if self.prediction_root is not None: posepred = pickle.load(open(self.predPosepath_list[index], "rb")) elif self.RANSACPose_root is not None: posepred = pickle.load(open(self.RANSACPose_list[index], "rb")) else: posepred = None inspred, posepred = self.pad_clip_ins(insmap=inspred, posepred=posepred) if not hasattr(self, 'deepv2dpred_root'): self.deepv2dpred_root = None if self.deepv2dpred_root is not None: depthpred_deepv2d = np.array( Image.open(self.deepv2dpredpath_list[index])).astype( np.float32) / 256.0 posepred_deepv2d = read_deepv2d_pose( self.deepv2dpredpose_list[index]) else: depthpred_deepv2d = None posepred_deepv2d = None if self.mdPred_root is not None: mdDepth_pred = np.array(Image.open( self.mdDepthpath_list[index])).astype(np.float32) / 256.0 else: mdDepth_pred = None if self.flowPred_root is not None: flowpred_RAFT, valid_flow = readFlowKITTI( self.flowpredpath_list[index]) else: flowpred_RAFT = None img1, img2, depth, depthvls, depthpred, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic = self.aug_crop( img1, img2, depth, depthvls, depthpred, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic) flowgt = self.get_gt_flow(depth=depth, valid=(inspred == 0) * (depth > 0), intrinsic=intrinsic, rel_pose=rel_pose) flowgt_vls = self.get_gt_flow(depth=depthvls, valid=(inspred == 0) * (depthvls > 0), intrinsic=intrinsic, rel_pose=rel_pose) if self.istrain and not self.muteaug: img1, img2 = self.colorjitter(img1, img2) data_blob = self.wrapup(img1=img1, img2=img2, flowgt=flowgt, flowgt_vls=flowgt_vls, depthmap=depth, depthvls=depthvls, depthpred=depthpred, depthpred_deepv2d=depthpred_deepv2d, mdDepth_pred=mdDepth_pred, intrinsic=intrinsic, insmap=inspred, flowpred=flowpred_RAFT, rel_pose=rel_pose, posepred=posepred, posepred_deepv2d=posepred_deepv2d, tag=self.entries[index], size_str=size_str) return data_blob