def __getitem__(self, index):
if not self.init_seed:
worker_info = torch.utils.data.get_worker_info()
if worker_info is not None:
torch.manual_seed(worker_info.id)
np.random.seed(worker_info.id)
random.seed(worker_info.id)
self.init_seed = True
index = index % len(self.image_list)
flow = frame_utils.read_gen(self.flow_list[index])
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)[..., :3]
img2 = np.array(img2).astype(np.uint8)[..., :3]
flow = np.array(flow).astype(np.float32)
if self.do_augument:
img1, img2, flow = self.augumentor(img1, img2, flow)
img1 = torch.from_numpy(img1).permute(2, 0, 1).float()
img2 = torch.from_numpy(img2).permute(2, 0, 1).float()
flow = torch.from_numpy(flow).permute(2, 0, 1).float()
valid = torch.ones_like(flow[0])
return img1, img2, flow, valid
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
intrinsic = copy.deepcopy(self.intrinsic_list[index])
RANSACPose = copy.deepcopy(self.RANSACPose_rec[index])
IMUPose = copy.deepcopy(self.IMUPose_rec[index])
if self.ins_root is not None:
inspred = np.array(Image.open(self.inspred_list[index])).astype(
np.int)
else:
inspred = None
mdDepth = np.array(Image.open(self.mdPred_list[index])).astype(
np.float32) / 256.0
stereogt = np.array(Image.open(self.stereoGt_list[index])).astype(
np.float32) / 256.0
flowpred_RAFT, _ = readFlowKITTI(self.flowPred_list[index])
flowgt, flowgt_valid = readFlowKITTI(self.flowGt_list[index])
data_blob = self.wrapup(img1=img1,
img2=img2,
intrinsic=intrinsic,
insmap=inspred,
mdDepth=mdDepth,
stereogt=stereogt,
flowpred_RAFT=flowpred_RAFT,
flowgt=flowgt,
flowgt_valid=flowgt_valid,
RANSACPose=RANSACPose,
IMUPose=IMUPose,
tag=self.entries_map[index])
return data_blob
def read_imgpair_flow_depth_instance_intrinsic(self, index):
img1 = np.array(read_gen(self.rgb1_paths[index])).astype(np.uint8)
img2 = np.array(read_gen(self.rgb2_paths[index])).astype(np.uint8)
flowmap = np.array(readFlowVRKitti(self.flow_paths[index])).astype(
np.float32)
depthmap = np.array(
cv2.imread(self.depth_paths[index], cv2.IMREAD_ANYCOLOR
| cv2.IMREAD_ANYDEPTH)).astype(np.float32)
depthmap = depthmap / 100
instancemap = np.array(Image.open(self.ins_paths[index]))
instancemap = np.array(instancemap).astype(np.int32) - 1
sceneidx, envn, frmidx = self.entries[index].split(' ')
frmidx = int(frmidx)
camidx = int(0)
scene_name = "Scene{}_{}".format(sceneidx.zfill(2), envn)
intrinsic_key = "{}_frm{}_cam{}".format(scene_name,
str(frmidx).zfill(5), camidx)
intrinsic = copy.deepcopy(self.intrinsic_dict[intrinsic_key])
return img1, img2, flowmap, depthmap, instancemap, intrinsic
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
intrinsic = copy.deepcopy(self.intrinsic_list[index])
if self.ins_root is not None:
inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int)
if self.banins:
inspred = inspred * 0
inspred = inspred.astype(np.int)
else:
inspred = None
if len(self.bsposepath_list[index]) == 0:
posepred_bs = np.eye(4)
posepred_bs = np.expand_dims(posepred_bs, axis=0)
posepred_bs = np.repeat(posepred_bs, axis=0, repeats=int(inspred.max() + 1))
else:
posepred_bs = pickle.load(open(self.bsposepath_list[index], "rb"))
mdDepth = np.array(Image.open(self.mdPred_list[index])).astype(np.float32) / 256.0
flowpred_RAFT, valid_flow = readFlowKITTI(self.flowPred_list[index])
data_blob = self.wrapup(img1=img1, img2=img2, intrinsic=intrinsic, insmap=inspred, mdDepth=mdDepth, flowpred_RAFT=flowpred_RAFT, posepred_bs=posepred_bs, tag=self.entries[index])
return data_blob
def __getitem__(self, index):
if self.is_test:
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)[..., :3]
img2 = np.array(img2).astype(np.uint8)[..., :3]
img1 = torch.from_numpy(img1).permute(2, 0, 1).float()
img2 = torch.from_numpy(img2).permute(2, 0, 1).float()
return img1, img2, self.extra_info[index]
if not self.init_seed:
worker_info = torch.utils.data.get_worker_info()
if worker_info is not None:
torch.manual_seed(worker_info.id)
np.random.seed(worker_info.id)
random.seed(worker_info.id)
self.init_seed = True
index = index % len(self.image_list)
valid = None
if self.sparse:
flow, valid = frame_utils.readFlowKITTI(self.flow_list[index])
else:
flow = frame_utils.read_gen(self.flow_list[index])
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
flow = np.array(flow).astype(np.float32)
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
# grayscale images
if len(img1.shape) == 2:
img1 = np.tile(img1[..., None], (1, 1, 3))
img2 = np.tile(img2[..., None], (1, 1, 3))
else:
img1 = img1[..., :3]
img2 = img2[..., :3]
if self.augmentor is not None:
if self.sparse:
img1, img2, flow, valid = self.augmentor(
img1, img2, flow, valid)
else:
img1, img2, flow = self.augmentor(img1, img2, flow)
img1 = torch.from_numpy(img1).permute(2, 0, 1).float()
img2 = torch.from_numpy(img2).permute(2, 0, 1).float()
flow = torch.from_numpy(flow).permute(2, 0, 1).float()
if valid is not None:
valid = torch.from_numpy(valid)
else:
valid = (flow[0].abs() < 1000) & (flow[1].abs() < 1000)
return img1, img2, flow, valid.float()
def validate_kitti_colorjitter(gpu, model, args, ngpus_per_node, eval_entries, iters=24):
""" Peform validation using the KITTI-2015 (train) split """
interval = np.floor(len(eval_entries) / ngpus_per_node).astype(np.int).item()
if gpu == ngpus_per_node - 1:
stidx = int(interval * gpu)
edidx = len(eval_entries)
else:
stidx = int(interval * gpu)
edidx = int(interval * (gpu + 1))
print("Initialize Instance on Gpu %d, from %d to %d, total %d" % (gpu, stidx, edidx, len(eval_entries)))
from tqdm import tqdm
model.eval()
model.cuda(gpu)
with torch.no_grad():
for val_id, entry in enumerate(tqdm(remove_dup(eval_entries[stidx : edidx]))):
seq, index = entry.split(' ')
index = int(index)
if os.path.exists(os.path.join(args.dataset, seq, 'image_02', 'data', "{}.png".format(str(index).zfill(10)))):
tmproot = args.dataset
else:
tmproot = args.odom_root
img1path = os.path.join(tmproot, seq, 'image_02', 'data', "{}.png".format(str(index).zfill(10)))
img2path = os.path.join(tmproot, seq, 'image_02', 'data', "{}.png".format(str(index + 1).zfill(10)))
if not os.path.exists(img2path):
img2path = img1path
image1 = frame_utils.read_gen(img1path)
image2 = frame_utils.read_gen(img2path)
image1 = np.array(image1).astype(np.uint8)
image2 = np.array(image2).astype(np.uint8)
image1 = torch.from_numpy(image1).permute([2, 0, 1]).float()
image2 = torch.from_numpy(image2).permute([2, 0, 1]).float()
svfold = os.path.join(args.exportroot, seq, 'image_02')
svpath = os.path.join(args.exportroot, seq, 'image_02', "{}.png".format(str(index).zfill(10)))
os.makedirs(svfold, exist_ok=True)
image1 = image1[None].cuda(gpu)
image2 = image2[None].cuda(gpu)
padder = InputPadder(image1.shape, mode='kitti')
image1, image2 = padder.pad(image1, image2)
flow_low, flow_pr = model(image1, image2, iters=iters, test_mode=True)
flow = padder.unpad(flow_pr[0]).cpu()
frame_utils.writeFlowKITTI(svpath, flow.permute(1, 2, 0).numpy())
# Image.fromarray(flow_viz.flow_to_image(flow.permute(1, 2, 0).numpy())).show()
# tensor2rgb(image1 / 255.0, viewind=0).show()
return
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
sequence = self.image_list[index][2]
frame = self.image_list[index][3]
img1 = np.array(img1).astype(np.uint8)[..., :3]
img2 = np.array(img2).astype(np.uint8)[..., :3]
img1 = torch.from_numpy(img1).permute(2, 0, 1).float()
img2 = torch.from_numpy(img2).permute(2, 0, 1).float()
return img1, img2, sequence, frame
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
intrinsic = copy.deepcopy(self.intrinsic_list[index])
if self.ins_root is not None:
inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int)
else:
inspred = None
mdDepth = np.array(Image.open(self.mdPred_list[index])).astype(np.float32) / 256.0
flowpred_RAFT, valid_flow = readFlowKITTI(self.flowPred_list[index])
data_blob = self.wrapup(img1=img1, img2=img2, intrinsic=intrinsic, insmap=inspred, mdDepth=mdDepth, flowpred_RAFT=flowpred_RAFT, tag=self.entries[index])
return data_blob
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
intrinsic = copy.deepcopy(self.intrinsic)
mdDepth = cv2.imread(self.mdPred_list[index], -1)
mdDepth = mdDepth.astype(np.float32) / 1000.0
gtDepth = cv2.imread(self.depthgt_list[index], -1)
gtDepth = gtDepth.astype(np.float32) / 1000.0
flowpred_RAFT, valid_flow = readFlowKITTI(self.flowPred_list[index])
data_blob = self.wrapup(img1=img1, img2=img2, intrinsic=intrinsic, mdDepth=mdDepth, gtDepth=gtDepth, flowpred_RAFT=flowpred_RAFT, tag=self.entries[index])
return data_blob
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
depth = np.array(Image.open(self.depth_list[index])).astype(
np.float32) / 256.0
depthvls = np.array(Image.open(self.depthvls_list[index])).astype(
np.float32) / 256.0
intrinsic = copy.deepcopy(self.intrinsic_list[index])
rel_pose = copy.deepcopy(self.pose_list[index])
inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int)
posepred = pickle.load(open(self.RANSACPose_list[index], "rb"))
inspred, posepred = self.pad_clip_ins(insmap=inspred,
posepred=posepred)
if self.mdPred_root is not None:
mdDepth_pred = np.array(Image.open(
self.mdDepthpath_list[index])).astype(np.float32) / 256.0
else:
mdDepth_pred = None
img1, img2, depth, depthvls, mdDepth_pred, inspred, intrinsic = self.aug_crop(
img1, img2, depth, depthvls, mdDepth_pred, inspred, intrinsic)
if self.istrain and not self.muteaug:
img1, img2 = self.colorjitter(img1, img2)
data_blob = self.wrapup(img1=img1,
img2=img2,
depthmap=depth,
depthvls=depthvls,
mdDepth_pred=mdDepth_pred,
intrinsic=intrinsic,
insmap=inspred,
rel_pose=rel_pose,
posepred=posepred,
tag=self.entries[index])
return data_blob
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
depth = cv2.imread(self.depth_list[index], -1) / 1000.0
intrinsic = copy.deepcopy(self.intrinsic)
if self.istrain:
np.random.seed(index + int(time.time()))
samp_rep = int(np.random.choice(list(range(self.nposs)), 1).item())
else:
samp_rep = int(0)
mdDepth_pred = cv2.imread(
self.mdDepthpath_list[index].format(str(samp_rep).zfill(3)),
-1) / 1000.0
posepred = pickle.load(
open(self.RANSACPose_list[index].format(str(samp_rep).zfill(3)),
"rb"))
flowpred, _ = frame_utils.readFlowKITTI(self.flowpred_list[index])
img1, img2, depth, mdDepth_pred, flowpred, intrinsic = self.aug_crop(
img1, img2, depth, mdDepth_pred, flowpred, intrinsic)
if self.istrain:
img1, img2 = self.colorjitter(img1, img2)
data_blob = self.wrapup(img1=img1,
img2=img2,
depthmap=depth,
mdDepth_pred=mdDepth_pred,
flowpred=flowpred,
intrinsic=intrinsic,
posepred=posepred,
tag=self.entries[index])
return data_blob
def validate_kitti_colorjitter(gpu,
model,
args,
ngpus_per_node,
eval_entries,
iters=24):
""" Peform validation using the KITTI-2015 (train) split """
interval = np.floor(len(eval_entries) / ngpus_per_node).astype(
np.int).item()
if gpu == ngpus_per_node - 1:
stidx = int(interval * gpu)
edidx = len(eval_entries)
else:
stidx = int(interval * gpu)
edidx = int(interval * (gpu + 1))
print("Initialize Instance on Gpu %d, from %d to %d, total %d" %
(gpu, stidx, edidx, len(eval_entries)))
from tqdm import tqdm
model.eval()
model.cuda(gpu)
with torch.no_grad():
for val_id, entry in enumerate(
tqdm(remove_dup(eval_entries[stidx:edidx]))):
seq, index = entry.split(' ')
index = int(index)
img1path = os.path.join(args.dataset, seq,
'rgb_{}.png'.format(str(index).zfill(5)))
img2path = os.path.join(
args.dataset, seq,
'rgb_{}.png'.format(str(index + 1).zfill(5)))
if not os.path.exists(img2path) and not os.path.exists(
img2path.replace('.png', '.jpg')):
img2path = img1path
if not os.path.exists(img1path):
img1path = img1path.replace('.png', '.jpg')
img2path = img2path.replace('.png', '.jpg')
image1 = frame_utils.read_gen(img1path)
image2 = frame_utils.read_gen(img2path)
image1 = np.array(image1).astype(np.uint8)
image2 = np.array(image2).astype(np.uint8)
image1 = torch.from_numpy(image1).permute([2, 0, 1]).float()
image2 = torch.from_numpy(image2).permute([2, 0, 1]).float()
svfold = os.path.join(
args.exportroot,
seq,
)
svpath = os.path.join(args.exportroot, seq,
'{}.png'.format(str(index).zfill(5)))
os.makedirs(svfold, exist_ok=True)
image1 = image1[None].cuda(gpu)
image2 = image2[None].cuda(gpu)
flow_low, flow_pr = model(image1,
image2,
iters=iters,
test_mode=True)
flow = flow_pr.squeeze(0)
flow_numpy = flow.cpu().permute(1, 2, 0).numpy()
frame_utils.writeFlowKITTI(svpath, flow_numpy)
if args.dovls:
vlsflow = Image.fromarray(flow_viz.flow_to_image(flow_numpy))
vlsrgb1 = tensor2rgb(image1 / 255.0, viewind=0)
vlsrgb2 = tensor2rgb(image2 / 255.0, viewind=0)
w, h = vlsrgb2.size
xx, yy = np.meshgrid(range(w), range(h), indexing='xy')
xxf = xx.flatten()
yyf = yy.flatten()
rndidx = np.random.randint(0, xxf.shape[0], 100)
xxf_rnd = xxf[rndidx]
yyf_rnd = yyf[rndidx]
flowx = flow_numpy[yyf_rnd, xxf_rnd, 0]
flowy = flow_numpy[yyf_rnd, xxf_rnd, 1]
fig = plt.figure(figsize=(12, 9))
plt.subplot(2, 1, 1)
plt.scatter(xxf_rnd, yyf_rnd, 1, 'r')
plt.imshow(vlsrgb1)
plt.subplot(2, 1, 2)
plt.scatter(flowx + xxf_rnd, flowy + yyf_rnd, 1, 'r')
plt.imshow(vlsrgb2)
plt.savefig(
os.path.join(
"/media/shengjie/disk1/Prediction/nyuv2_flow_vls",
"{}_{}.jpg".format(seq,
str(index).zfill(5))))
plt.close()
# combined_left = np.concatenate([np.array(vlsrgb1), np.array(vlsrgb2)], axis=0)
# combined_right = np.concatenate([np.array(vlsflow), np.array(vlsflow)], axis=0)
# combined = np.concatenate([combined_left, combined_right], axis=1)
# vls_sv_root = os.makedirs("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", exist_ok=True)
# Image.fromarray(combined).save(os.path.join("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", "{}_{}.jpg".format(seq, str(index).zfill(5))))
return
def __getitem__(self, index):
if self.is_test:
frame_id = self.image_list[index][0]
frame_id = frame_id.split('/')[-1]
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)[..., :3]
img2 = np.array(img2).astype(np.uint8)[..., :3]
img1 = torch.from_numpy(img1).permute(2, 0, 1).float()
img2 = torch.from_numpy(img2).permute(2, 0, 1).float()
return img1, img2, frame_id
else:
if not self.init_seed:
worker_info = torch.utils.data.get_worker_info()
if worker_info is not None:
np.random.seed(worker_info.id)
random.seed(worker_info.id)
self.init_seed = True
index = index % len(self.image_list)
frame_id = self.image_list[index][0]
frame_id = frame_id.split('/')[-1]
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
flow, valid = frame_utils.readFlowKITTI(self.flow_list[index])
img1 = np.array(img1).astype(np.uint8)[..., :3]
img2 = np.array(img2).astype(np.uint8)[..., :3]
if self.do_augument:
img1, img2, flow, valid = self.augumentor(
img1, img2, flow, valid)
img1 = torch.from_numpy(img1).permute(2, 0, 1).float()
img2 = torch.from_numpy(img2).permute(2, 0, 1).float()
flow = torch.from_numpy(flow).permute(2, 0, 1).float()
valid = torch.from_numpy(valid).float()
if self.do_pad:
ht, wd = img1.shape[1:]
pad_ht = (((ht // 8) + 1) * 8 - ht) % 8
pad_wd = (((wd // 8) + 1) * 8 - wd) % 8
pad_ht1 = [0, pad_ht]
pad_wd1 = [pad_wd // 2, pad_wd - pad_wd // 2]
pad = pad_wd1 + pad_ht1
img1 = img1.view(1, 3, ht, wd)
img2 = img2.view(1, 3, ht, wd)
flow = flow.view(1, 2, ht, wd)
valid = valid.view(1, 1, ht, wd)
img1 = torch.nn.functional.pad(img1, pad, mode='replicate')
img2 = torch.nn.functional.pad(img2, pad, mode='replicate')
flow = torch.nn.functional.pad(flow,
pad,
mode='constant',
value=0)
valid = torch.nn.functional.pad(valid,
pad,
mode='replicate',
value=0)
img1 = img1.view(3, ht + pad_ht, wd + pad_wd)
img2 = img2.view(3, ht + pad_ht, wd + pad_wd)
flow = flow.view(2, ht + pad_ht, wd + pad_wd)
valid = valid.view(ht + pad_ht, wd + pad_wd)
if self.is_test:
return img1, img2, flow, valid, frame_id
return img1, img2, flow, valid
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
depthvls = np.array(Image.open(self.depthvls_list[index])).astype(np.float32) / 256.0
intrinsic = copy.deepcopy(self.intrinsic_list[index])
rel_pose = copy.deepcopy(self.pose_list[index])
inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int)
if self.RANSACPose_root is not None:
np.random.seed(index + int(time.time()))
rep_idces = np.random.choice(list(range(self.npossibility)), self.num_samples, replace=False)
# rep_idces = np.array([0, 1, 2, 3])
# rep_idces = np.array([0])
posepred = list()
ang_decps_l = list()
scl_decps_l = list()
mvd_decps_l = list()
for rep_idx in rep_idces:
posepred_cur = pickle.load(open(self.RANSACPose_list[index].format(str(rep_idx).zfill(3)), "rb"))
for l in self.linlogdedge:
scaledpose = copy.deepcopy(posepred_cur)
for kk in range(scaledpose.shape[0]):
tmpscale = np.sqrt(np.sum(scaledpose[kk, 0:3, 3] ** 2) + 1e-12)
adjustedscale = np.exp(np.log(tmpscale) + l)
scaledpose[kk, 0:3, 3] = scaledpose[kk, 0:3, 3] / tmpscale * adjustedscale
posepred.append(scaledpose)
posepred_decps = copy.deepcopy(posepred_cur)
ang_decps = np.zeros([posepred_decps.shape[0], 3])
scl_decps = np.zeros([posepred_decps.shape[0], 1])
mvd_decps = np.zeros([posepred_decps.shape[0], 3])
for k in range(posepred_decps.shape[0]):
ang_decps[k, :] = self.rot2ang(posepred_decps[k])
tmps = np.sum(posepred_decps[k, 0:3, 3] ** 2)
if tmps == 0:
scl_decps[k, 0] = 1e-6
mvd_decps[k, :] = np.array([0, 0, -1])
else:
scl_decps[k, 0] = np.sqrt(tmps)
mvd_decps[k, :] = posepred_decps[k, 0:3, 3] / scl_decps[k, 0]
ang_decps_l.append(ang_decps)
scl_decps_l.append(scl_decps)
mvd_decps_l.append(mvd_decps)
posepred = np.stack(posepred, axis=0)
ang_decps = np.stack(ang_decps_l, axis=0)
scl_decps = np.stack(scl_decps_l, axis=0)
mvd_decps = np.stack(mvd_decps_l, axis=0)
else:
posepred = None
ang_decps = None
scl_decps = None
mvd_decps = None
inspred, posepred, ang_decps_pad, scl_decps_pad, mvd_decps_pad = self.pad_clip_ins(insmap=inspred, posepred=posepred, ang_decps=ang_decps, scl_decps=scl_decps, mvd_decps=mvd_decps)
if not hasattr(self, 'deepv2dpred_root'):
self.deepv2dpred_root = None
if self.depth_root is not None:
depth = np.array(Image.open(self.depth_list[index])).astype(np.float32) / 256.0
else:
depth = None
if self.deepv2dpred_root is not None:
depthpred_deepv2d = np.array(Image.open(self.deepv2dpredpath_list[index])).astype(np.float32) / 256.0
posepred_deepv2d = read_deepv2d_pose(self.deepv2dpredpose_list[index])
else:
depthpred_deepv2d = None
posepred_deepv2d = None
if self.mdPred_root is not None:
mdDepth_pred = np.array(Image.open(self.mdDepthpath_list[index])).astype(np.float32) / 256.0
else:
mdDepth_pred = None
if self.flowPred_root is not None:
flowpred_RAFT, valid_flow = readFlowKITTI(self.flowpredpath_list[index])
else:
flowpred_RAFT = None
img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic = self.aug_crop(img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic)
if self.depth_root is not None:
flowgt = self.get_gt_flow(depth=depth, valid=(inspred==0) * (depth>0), intrinsic=intrinsic, rel_pose=rel_pose)
else:
flowgt = None
flowgt_vls = self.get_gt_flow(depth=depthvls, valid=(inspred==0) * (depthvls>0), intrinsic=intrinsic, rel_pose=rel_pose)
if self.istrain and not self.muteaug:
img1, img2 = self.colorjitter(img1, img2)
entry = self.entries[index]
seq, frmidx = entry.split(' ')
frmidx = int(frmidx)
seq = seq.split('/')[1]
IMUposes_seg = list()
RANSACposes_seg = list()
IMU_poses_reced = self.IMU_poses_rec[seq]
RANSAC_poses_reced = self.RANSAC_poses_rec[seq]
for k in range(frmidx - self.span, frmidx + self.span):
if k < 0:
IMUposes_seg.append(np.eye(4))
RANSACposes_seg.append(np.eye(4))
elif k >= len(IMU_poses_reced):
IMUposes_seg.append(np.eye(4))
RANSACposes_seg.append(np.eye(4))
else:
IMUposes_seg.append(IMU_poses_reced[k])
RANSACposes_seg.append(RANSAC_poses_reced[k])
# Forward move
IMUlocations1 = list()
leftarrs1 = list()
rightarrs1 = list()
accumM = np.eye(4)
leftarr = np.eye(4)
rightarr = np.eye(4)
for k in range(len(IMUposes_seg)):
accumM = IMUposes_seg[k] @ accumM
if k < self.span:
rightarr = RANSACposes_seg[k] @ rightarr
if k > self.span:
leftarr = RANSACposes_seg[k] @ leftarr
if k >= self.span:
IMUlocations1.append(np.linalg.inv(accumM)[0:3, 3:4])
leftarrs1.append(leftarr)
rightarrs1.append(rightarr)
# print(np.sum(np.abs(np.linalg.inv(leftarr @ rel_pose @ rightarr)[0:3, 3] - IMUlocations1[k-self.span])))
IMUlocations2 = list()
leftarrs2 = list()
rightarrs2 = list()
accumM = np.eye(4)
leftarr = np.eye(4)
rightarr = np.eye(4)
for k in range(len(IMUposes_seg)):
accumM = np.linalg.inv(IMUposes_seg[-k]) @ accumM
if k < self.span:
rightarr = np.linalg.inv(RANSACposes_seg[-k]) @ rightarr
if k > self.span:
leftarr = np.linalg.inv(RANSACposes_seg[-k]) @ leftarr
if k >= self.span:
IMUlocations2.append(np.linalg.inv(accumM)[0:3, 3:4])
leftarrs2.append(leftarr)
rightarrs2.append(rightarr)
# print(np.sum(np.abs(np.linalg.inv(leftarr @ np.linalg.inv(rel_pose) @ rightarr)[0:3, 3] - IMUlocations2[k-self.span])))
IMUlocations1 = np.stack(IMUlocations1)
leftarrs1 = np.stack(leftarrs1)
rightarrs1 = np.stack(rightarrs1)
IMUlocations2 = np.stack(IMUlocations2)
leftarrs2 = np.stack(leftarrs2)
rightarrs2 = np.stack(rightarrs2)
data_blob = self.wrapup(img1=img1, img2=img2, flowgt=flowgt, flowgt_vls=flowgt_vls, depthmap=depth, depthvls=depthvls,
depthpred_deepv2d=depthpred_deepv2d, mdDepth_pred=mdDepth_pred, intrinsic=intrinsic, insmap=inspred, flowpred=flowpred_RAFT, rel_pose=rel_pose,
posepred=posepred, posepred_deepv2d=posepred_deepv2d, ang_decps_pad=ang_decps_pad, scl_decps_pad=scl_decps_pad, mvd_decps_pad=mvd_decps_pad, tag=self.entries[index],
IMUlocations1=IMUlocations1, leftarrs1=leftarrs1, rightarrs1=rightarrs1, IMUlocations2=IMUlocations2, leftarrs2=leftarrs2, rightarrs2=rightarrs2)
return data_blob
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
depthvls = np.array(Image.open(self.depthvls_list[index])).astype(
np.float32) / 256.0
intrinsic = copy.deepcopy(self.intrinsic_list[index])
rel_pose = copy.deepcopy(self.pose_list[index])
inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int)
if self.RANSACPose_root is not None:
np.random.seed(index + int(time.time()))
if self.num_samples == 1:
rep_idces = [0]
elif self.num_samples == 2:
rep_idces = [0, 1]
else:
rep_idces = [0, 1, 2, 3]
# rep_idces = [0, 0, 0, 0]
posepred = list()
ang_decps_l = list()
scl_decps_l = list()
mvd_decps_l = list()
for rep_idx in rep_idces:
posepred_cur = pickle.load(
open(
self.RANSACPose_list[index].format(
str(rep_idx).zfill(3)), "rb"))
for l in self.linlogdedge:
scaledpose = copy.deepcopy(posepred_cur)
for kk in range(scaledpose.shape[0]):
tmpscale = np.sqrt(
np.sum(scaledpose[kk, 0:3, 3]**2) + 1e-12)
adjustedscale = np.exp(np.log(tmpscale) + l)
scaledpose[kk, 0:3, 3] = scaledpose[
kk, 0:3, 3] / tmpscale * adjustedscale
posepred.append(scaledpose)
posepred_decps = copy.deepcopy(posepred_cur)
ang_decps = np.zeros([posepred_decps.shape[0], 3])
scl_decps = np.zeros([posepred_decps.shape[0], 1])
mvd_decps = np.zeros([posepred_decps.shape[0], 3])
for k in range(posepred_decps.shape[0]):
ang_decps[k, :] = self.rot2ang(posepred_decps[k])
tmps = np.sum(posepred_decps[k, 0:3, 3]**2)
if tmps == 0:
scl_decps[k, 0] = 1e-6
mvd_decps[k, :] = np.array([0, 0, -1])
else:
scl_decps[k, 0] = np.sqrt(tmps)
mvd_decps[k, :] = posepred_decps[k, 0:3,
3] / scl_decps[k, 0]
ang_decps_l.append(ang_decps)
scl_decps_l.append(scl_decps)
mvd_decps_l.append(mvd_decps)
posepred = np.stack(posepred, axis=0)
ang_decps = np.stack(ang_decps_l, axis=0)
scl_decps = np.stack(scl_decps_l, axis=0)
mvd_decps = np.stack(mvd_decps_l, axis=0)
else:
posepred = None
ang_decps = None
scl_decps = None
mvd_decps = None
relpose_odom_gt = copy.deepcopy(self.gt_relpose_rec[index])
inspred, posepred, ang_decps_pad, scl_decps_pad, mvd_decps_pad = self.pad_clip_ins(
insmap=inspred,
posepred=posepred,
ang_decps=ang_decps,
scl_decps=scl_decps,
mvd_decps=mvd_decps)
if not hasattr(self, 'deepv2dpred_root'):
self.deepv2dpred_root = None
if self.depth_root is not None:
depth = np.array(Image.open(self.depth_list[index])).astype(
np.float32) / 256.0
else:
depth = None
if self.deepv2dpred_root is not None:
depthpred_deepv2d = np.array(
Image.open(self.deepv2dpredpath_list[index])).astype(
np.float32) / 256.0
posepred_deepv2d = read_deepv2d_pose(
self.deepv2dpredpose_list[index])
else:
depthpred_deepv2d = None
posepred_deepv2d = None
if self.mdPred_root is not None:
mdDepth_pred = np.array(Image.open(
self.mdDepthpath_list[index])).astype(np.float32) / 256.0
else:
mdDepth_pred = None
if self.flowPred_root is not None:
flowpred_RAFT, valid_flow = readFlowKITTI(
self.flowpredpath_list[index])
else:
flowpred_RAFT = None
img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic = self.aug_crop(
img1, img2, depth, depthvls, depthpred_deepv2d, mdDepth_pred,
inspred, flowpred_RAFT, intrinsic)
if self.depth_root is not None:
flowgt = self.get_gt_flow(depth=depth,
valid=(inspred == 0) * (depth > 0),
intrinsic=intrinsic,
rel_pose=rel_pose)
else:
flowgt = None
flowgt_vls = self.get_gt_flow(depth=depthvls,
valid=(inspred == 0) * (depthvls > 0),
intrinsic=intrinsic,
rel_pose=rel_pose)
if self.istrain and not self.muteaug:
img1, img2 = self.colorjitter(img1, img2)
data_blob = self.wrapup(img1=img1,
img2=img2,
flowgt=flowgt,
flowgt_vls=flowgt_vls,
depthmap=depth,
depthvls=depthvls,
depthpred_deepv2d=depthpred_deepv2d,
mdDepth_pred=mdDepth_pred,
intrinsic=intrinsic,
insmap=inspred,
flowpred=flowpred_RAFT,
rel_pose=rel_pose,
posepred=posepred,
posepred_deepv2d=posepred_deepv2d,
relpose_odom_gt=relpose_odom_gt,
ang_decps_pad=ang_decps_pad,
scl_decps_pad=scl_decps_pad,
mvd_decps_pad=mvd_decps_pad,
tag=self.entries[index])
return data_blob
def validate_kitti_colorjitter(gpu,
model,
args,
ngpus_per_node,
eval_entries,
iters=24):
""" Peform validation using the KITTI-2015 (train) split """
stidx = 0
edidx = len(eval_entries)
print("Initialize Instance on Gpu %d, from %d to %d, total %d" %
(gpu, stidx, edidx, len(eval_entries)))
from tqdm import tqdm
model.eval()
model.cuda(gpu)
totnum = 0
dr = 0
with torch.no_grad():
for val_id, entry in enumerate(remove_dup(eval_entries[stidx:edidx])):
seq, index = entry.split(' ')
index = int(index)
if os.path.exists(
os.path.join(args.dataset, seq, 'image_02', 'data',
"{}.png".format(str(index).zfill(10)))):
tmproot = args.dataset
else:
tmproot = args.odom_root
img1path = os.path.join(tmproot, seq, 'image_02', 'data',
"{}.png".format(str(index).zfill(10)))
img2path = os.path.join(tmproot, seq, 'image_02', 'data',
"{}.png".format(str(index + 1).zfill(10)))
if not os.path.exists(img2path):
img2path = img1path
image1 = frame_utils.read_gen(img1path)
image2 = frame_utils.read_gen(img2path)
image1 = np.array(image1).astype(np.uint8)
image2 = np.array(image2).astype(np.uint8)
h, w, _ = image1.shape
crph = h - args.evalheight
image1 = image1[crph:h, 0:args.evalwidth, :]
image2 = image2[crph:h, 0:args.evalwidth, :]
image1 = torch.from_numpy(image1).permute([2, 0, 1]).float()
image2 = torch.from_numpy(image2).permute([2, 0, 1]).float()
image1 = image1[None].cuda(gpu)
image2 = image2[None].cuda(gpu)
# padder = InputPadder(image1.shape, mode='kitti')
# image1, image2 = padder.pad(image1, image2)
st = time.time()
flow_low, flow_pr = model(image1, image2, iters=24, test_mode=True)
dr += time.time() - st
totnum += 1
print("%d Samples, Ave sec/frame: %f, Mem: %f Gb" %
(totnum, dr / totnum,
float(torch.cuda.memory_allocated() / 1024 / 1024 / 1024)))
return
def __getitem__(self, index):
img1 = frame_utils.read_gen(self.image_list[index][0])
img2 = frame_utils.read_gen(self.image_list[index][1])
img1 = np.array(img1).astype(np.uint8)
img2 = np.array(img2).astype(np.uint8)
h, w, _ = img1.shape
size_str = "{}_{}".format(str(h), str(w))
if not self.bandepth:
depth = np.array(Image.open(self.depth_list[index])).astype(
np.float32) / 256.0
depthvls = np.array(Image.open(self.depthvls_list[index])).astype(
np.float32) / 256.0
else:
depth = np.zeros([h, w])
depthvls = np.zeros([h, w])
intrinsic = copy.deepcopy(self.intrinsic_list[index])
rel_pose = copy.deepcopy(self.pose_list[index])
inspred = np.array(Image.open(self.inspred_list[index])).astype(np.int)
if self.prediction_root is not None:
depthpred = np.array(Image.open(
self.predDepthpath_list[index])).astype(np.float32) / 256.0
else:
depthpred = None
assert not (self.prediction_root is not None
and self.RANSACPose_root is not None)
if self.prediction_root is not None:
posepred = pickle.load(open(self.predPosepath_list[index], "rb"))
elif self.RANSACPose_root is not None:
posepred = pickle.load(open(self.RANSACPose_list[index], "rb"))
else:
posepred = None
inspred, posepred = self.pad_clip_ins(insmap=inspred,
posepred=posepred)
if not hasattr(self, 'deepv2dpred_root'):
self.deepv2dpred_root = None
if self.deepv2dpred_root is not None:
depthpred_deepv2d = np.array(
Image.open(self.deepv2dpredpath_list[index])).astype(
np.float32) / 256.0
posepred_deepv2d = read_deepv2d_pose(
self.deepv2dpredpose_list[index])
else:
depthpred_deepv2d = None
posepred_deepv2d = None
if self.mdPred_root is not None:
mdDepth_pred = np.array(Image.open(
self.mdDepthpath_list[index])).astype(np.float32) / 256.0
else:
mdDepth_pred = None
if self.flowPred_root is not None:
flowpred_RAFT, valid_flow = readFlowKITTI(
self.flowpredpath_list[index])
else:
flowpred_RAFT = None
img1, img2, depth, depthvls, depthpred, depthpred_deepv2d, mdDepth_pred, inspred, flowpred_RAFT, intrinsic = self.aug_crop(
img1, img2, depth, depthvls, depthpred, depthpred_deepv2d,
mdDepth_pred, inspred, flowpred_RAFT, intrinsic)
flowgt = self.get_gt_flow(depth=depth,
valid=(inspred == 0) * (depth > 0),
intrinsic=intrinsic,
rel_pose=rel_pose)
flowgt_vls = self.get_gt_flow(depth=depthvls,
valid=(inspred == 0) * (depthvls > 0),
intrinsic=intrinsic,
rel_pose=rel_pose)
if self.istrain and not self.muteaug:
img1, img2 = self.colorjitter(img1, img2)
data_blob = self.wrapup(img1=img1,
img2=img2,
flowgt=flowgt,
flowgt_vls=flowgt_vls,
depthmap=depth,
depthvls=depthvls,
depthpred=depthpred,
depthpred_deepv2d=depthpred_deepv2d,
mdDepth_pred=mdDepth_pred,
intrinsic=intrinsic,
insmap=inspred,
flowpred=flowpred_RAFT,
rel_pose=rel_pose,
posepred=posepred,
posepred_deepv2d=posepred_deepv2d,
tag=self.entries[index],
size_str=size_str)
return data_blob
