def write_vls_eval(self, data_blob, outputs, tagname, step):
img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
inputdepth = tensor2disp(1 / data_blob['depthmap'], vmax=1, viewind=0)
depthpredvls = tensor2disp(1 / outputs[('depth', 2)],
vmax=1,
viewind=0)
flowvls = flow_to_image(
outputs[('flowpred', 2)][0].detach().cpu().permute([1, 2,
0]).numpy(),
rad_max=10)
imgrecon = tensor2rgb(outputs[('reconImg', 2)], viewind=0)
img_val_up = np.concatenate([np.array(img1), np.array(img2)], axis=1)
img_val_mid2 = np.concatenate(
[np.array(inputdepth),
np.array(depthpredvls)], axis=1)
img_val_mid3 = np.concatenate(
[np.array(imgrecon), np.array(flowvls)], axis=1)
img_val = np.concatenate([
np.array(img_val_up),
np.array(img_val_mid2),
np.array(img_val_mid3)
],
axis=0)
self.writer.add_image('{}_predvls'.format(tagname),
(torch.from_numpy(img_val).float() /
255).permute([2, 0, 1]), step)
X = self.vls_sampling(img1, img2, data_blob['depthmap'],
data_blob['flowpred'], outputs)
self.writer.add_image('{}_X'.format(tagname),
(torch.from_numpy(X).float() / 255).permute(
[2, 0, 1]), step)
def write_vls(self, data_blob, outputs, step):
img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
insmap = data_blob['insmap'][0].squeeze().numpy()
insvls = vls_ins(img1, insmap)
depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'],
vmax=0.15,
viewind=0)
imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0)
attentionmap = tensor2disp(outputs['attention'],
percentile=95,
viewind=0)
img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1)
img_val_mid = np.concatenate(
[np.array(depthpredvls),
np.array(attentionmap)], axis=1)
img_val = np.concatenate(
[np.array(img_val_up), np.array(img_val_mid)], axis=0)
self.writer.add_image('predvls', (torch.from_numpy(img_val).float() /
255).permute([2, 0, 1]), step)
X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'],
outputs)
self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute(
[2, 0, 1]), step)
def write_vls(self, data_blob, outputs, flowselector, step):
img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
insmap = data_blob['insmap'][0].squeeze().numpy()
figmask_flow = tensor2disp(flowselector, vmax=1, viewind=0)
insvls = vls_ins(img1, insmap)
depthpredvls = tensor2disp(1 / outputs[('depth', 2)],
vmax=0.15,
viewind=0)
flowvls = flow_to_image(
outputs[('flowpred', 2)][0].detach().cpu().permute([1, 2,
0]).numpy(),
rad_max=10)
imgrecon = tensor2rgb(outputs[('reconImg', 2)], viewind=0)
img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1)
img_val_mid2 = np.concatenate(
[np.array(depthpredvls),
np.array(figmask_flow)], axis=1)
img_val_mid3 = np.concatenate(
[np.array(imgrecon), np.array(flowvls)], axis=1)
img_val = np.concatenate([
np.array(img_val_up),
np.array(img_val_mid2),
np.array(img_val_mid3)
],
axis=0)
self.writer.add_image('predvls', (torch.from_numpy(img_val).float() /
255).permute([2, 0, 1]), step)
def write_vls_eval(self, data_blob, outputs, tagname, step):
img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
insmap = data_blob['insmap'][0].squeeze().numpy()
insvls = vls_ins(img1, insmap)
depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'], vmax=0.15, viewind=0)
imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0)
img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1)
img_val_mid = np.concatenate([np.array(depthpredvls), np.array(imgrecon)], axis=1)
img_val = np.concatenate([np.array(img_val_up), np.array(img_val_mid)], axis=0)
self.writer.add_image('{}_predvls'.format(tagname), (torch.from_numpy(img_val).float() / 255).permute([2, 0, 1]), step)
def write_vls(self, data_blob, outputs, step):
img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
insmap = data_blob['insmap'][0].squeeze().numpy()
insvls = vls_ins(img1, insmap)
depthpredvls = tensor2disp(1 / data_blob['mdDepth_pred'],
vmax=0.15,
viewind=0)
imgrecon = tensor2rgb(outputs[('img1_recon', 2)][:, -1], viewind=0)
flow_RAFT = Image.fromarray(
flow_to_image(data_blob['flowpred'][0].permute([1, 2,
0]).cpu().numpy()))
flow_pred = Image.fromarray(
flow_to_image(outputs[('flowpred',
2)][0].permute([1, 2,
0]).detach().cpu().numpy()))
img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1)
img_val_mid = np.concatenate(
[np.array(depthpredvls),
np.array(imgrecon)], axis=1)
img_val_flow = np.concatenate(
[np.array(flow_RAFT), np.array(flow_pred)], axis=1)
img_val = np.concatenate([
np.array(img_val_up),
np.array(img_val_mid),
np.array(img_val_flow)
],
axis=0)
self.writer.add_image('predvls', (torch.from_numpy(img_val).float() /
255).permute([2, 0, 1]), step)
X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'],
outputs)
self.writer.add_image('X', (torch.from_numpy(X).float() / 255).permute(
[2, 0, 1]), step)
X = self.vls_objmvment(np.array(insvls), data_blob['insmap'],
data_blob['posepred'])
self.writer.add_image('objmvment', (torch.from_numpy(X).float() /
255).permute([2, 0, 1]), step)
def write_vls_eval(self, data_blob, outputs, tagname, step):
img1 = data_blob['img1'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
img2 = data_blob['img2'][0].permute([1, 2, 0]).numpy().astype(np.uint8)
insmap = data_blob['insmap'][0].squeeze().numpy()
insvls = vls_ins(img1, insmap)
depthpredvls = tensor2disp(1 / outputs['depth_predictions'][-1],
vmax=0.15,
viewind=0)
depthgtvls = tensor2disp(1 / data_blob['depthmap'],
vmax=0.15,
viewind=0)
flowvls = flow_to_image(outputs['flowpred'][0].detach().cpu().permute(
[1, 2, 0]).numpy(),
rad_max=50)
imgrecon = tensor2rgb(outputs['img1_recon'], viewind=0)
img_val_up = np.concatenate([np.array(insvls), np.array(img2)], axis=1)
img_val_mid2 = np.concatenate(
[np.array(depthpredvls),
np.array(depthgtvls)], axis=1)
img_val_mid3 = np.concatenate(
[np.array(imgrecon), np.array(flowvls)], axis=1)
img_val = np.concatenate([
np.array(img_val_up),
np.array(img_val_mid2),
np.array(img_val_mid3)
],
axis=0)
self.writer.add_image('{}_predvls'.format(tagname),
(torch.from_numpy(img_val).float() /
255).permute([2, 0, 1]), step)
X = self.vls_sampling(np.array(insvls), img2, data_blob['depthvls'],
data_blob['flowmap'], data_blob['insmap'],
outputs)
self.writer.add_image('{}_X'.format(tagname),
(torch.from_numpy(X).float() / 255).permute(
[2, 0, 1]), step)
def validate_kitti_colorjitter(gpu,
model,
args,
ngpus_per_node,
eval_entries,
iters=24):
""" Peform validation using the KITTI-2015 (train) split """
interval = np.floor(len(eval_entries) / ngpus_per_node).astype(
np.int).item()
if gpu == ngpus_per_node - 1:
stidx = int(interval * gpu)
edidx = len(eval_entries)
else:
stidx = int(interval * gpu)
edidx = int(interval * (gpu + 1))
print("Initialize Instance on Gpu %d, from %d to %d, total %d" %
(gpu, stidx, edidx, len(eval_entries)))
from tqdm import tqdm
model.eval()
model.cuda(gpu)
with torch.no_grad():
for val_id, entry in enumerate(
tqdm(remove_dup(eval_entries[stidx:edidx]))):
seq, index = entry.split(' ')
index = int(index)
img1path = os.path.join(args.dataset, seq,
'rgb_{}.png'.format(str(index).zfill(5)))
img2path = os.path.join(
args.dataset, seq,
'rgb_{}.png'.format(str(index + 1).zfill(5)))
if not os.path.exists(img2path) and not os.path.exists(
img2path.replace('.png', '.jpg')):
img2path = img1path
if not os.path.exists(img1path):
img1path = img1path.replace('.png', '.jpg')
img2path = img2path.replace('.png', '.jpg')
image1 = frame_utils.read_gen(img1path)
image2 = frame_utils.read_gen(img2path)
image1 = np.array(image1).astype(np.uint8)
image2 = np.array(image2).astype(np.uint8)
image1 = torch.from_numpy(image1).permute([2, 0, 1]).float()
image2 = torch.from_numpy(image2).permute([2, 0, 1]).float()
svfold = os.path.join(
args.exportroot,
seq,
)
svpath = os.path.join(args.exportroot, seq,
'{}.png'.format(str(index).zfill(5)))
os.makedirs(svfold, exist_ok=True)
image1 = image1[None].cuda(gpu)
image2 = image2[None].cuda(gpu)
flow_low, flow_pr = model(image1,
image2,
iters=iters,
test_mode=True)
flow = flow_pr.squeeze(0)
flow_numpy = flow.cpu().permute(1, 2, 0).numpy()
frame_utils.writeFlowKITTI(svpath, flow_numpy)
if args.dovls:
vlsflow = Image.fromarray(flow_viz.flow_to_image(flow_numpy))
vlsrgb1 = tensor2rgb(image1 / 255.0, viewind=0)
vlsrgb2 = tensor2rgb(image2 / 255.0, viewind=0)
w, h = vlsrgb2.size
xx, yy = np.meshgrid(range(w), range(h), indexing='xy')
xxf = xx.flatten()
yyf = yy.flatten()
rndidx = np.random.randint(0, xxf.shape[0], 100)
xxf_rnd = xxf[rndidx]
yyf_rnd = yyf[rndidx]
flowx = flow_numpy[yyf_rnd, xxf_rnd, 0]
flowy = flow_numpy[yyf_rnd, xxf_rnd, 1]
fig = plt.figure(figsize=(12, 9))
plt.subplot(2, 1, 1)
plt.scatter(xxf_rnd, yyf_rnd, 1, 'r')
plt.imshow(vlsrgb1)
plt.subplot(2, 1, 2)
plt.scatter(flowx + xxf_rnd, flowy + yyf_rnd, 1, 'r')
plt.imshow(vlsrgb2)
plt.savefig(
os.path.join(
"/media/shengjie/disk1/Prediction/nyuv2_flow_vls",
"{}_{}.jpg".format(seq,
str(index).zfill(5))))
plt.close()
# combined_left = np.concatenate([np.array(vlsrgb1), np.array(vlsrgb2)], axis=0)
# combined_right = np.concatenate([np.array(vlsflow), np.array(vlsflow)], axis=0)
# combined = np.concatenate([combined_left, combined_right], axis=1)
# vls_sv_root = os.makedirs("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", exist_ok=True)
# Image.fromarray(combined).save(os.path.join("/media/shengjie/disk1/Prediction/nyuv2_flow_vls", "{}_{}.jpg".format(seq, str(index).zfill(5))))
return
def validate_RANSAC_scalecontrast(args,
eval_loader,
group,
num_frames=5,
dovls=False):
gpu = args.gpu
eval_metrics = {
'pose_RANSAC_mdscale': torch.zeros(1).cuda(device=gpu),
'pose_RANSAC': torch.zeros(num_frames).cuda(device=gpu),
'pixelnum': torch.zeros(1).cuda(device=gpu),
'framenum': torch.zeros(1).cuda(device=gpu)
}
for val_id, data_blob in enumerate(tqdm(eval_loader)):
image1 = data_blob['img1'].cuda(gpu)
image2 = data_blob['img2'].cuda(gpu)
insmap = data_blob['insmap'].cuda(gpu)
intrinsic = data_blob['intrinsic'].cuda(gpu)
depthgt = data_blob['depthmap'].cuda(gpu)
valid_flow = data_blob['valid_flow'].cuda(gpu) == 1
tagname = data_blob['tag'][0]
flowgt_stereo = data_blob['flowgt_stereo'].cuda(gpu)
posepred_deepv2d = data_blob['posepred_deepv2d'].cuda(gpu)
flowpred = data_blob['flowpred'].cuda(gpu)
# mdDepth_pred = data_blob['mdDepth_pred'].cuda(gpu)
mdDepth_pred = data_blob['depthpred_deepv2d'].cuda(gpu)
mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt()
val = (mag < MAX_FLOW) * (insmap == 0) * valid_flow * (depthgt > 0)
# R, t, scale, scale_md, flow_sel_mag, istatic = inf_pose_flow(image1, image2, flowpred, insmap, depthgt, mdDepth_pred, intrinsic, int(val_id + args.gpu * 100000), gradComputer=None)
R, t, scale, scale_md, flow_sel_mag, istatic = inf_pose_flow(
image1,
image2,
flowpred,
insmap,
depthgt,
mdDepth_pred,
intrinsic,
int(val_id + args.gpu * 1000 + 10),
gradComputer=None)
pose_RANSAC = torch.clone(posepred_deepv2d)
pose_RANSAC_mdscale = torch.clone(posepred_deepv2d)
if istatic:
pose_RANSAC[0, 0, :, :] = torch.eye(4, device=intrinsic.device)
pose_RANSAC_mdscale[0,
0, :, :] = torch.eye(4,
device=intrinsic.device)
else:
pose_RANSAC[0, 0, 0:3, 0:3] = torch.from_numpy(R).float().cuda(
intrinsic.device)
pose_RANSAC[0, 0, 0:3, 3:4] = torch.from_numpy(
t * scale).float().cuda(intrinsic.device)
pose_RANSAC_mdscale[0, 0, 0:3,
0:3] = torch.from_numpy(R).float().cuda(
intrinsic.device)
pose_RANSAC_mdscale[0, 0, 0:3, 3:4] = torch.from_numpy(
t * scale_md).float().cuda(intrinsic.device)
poses = {
'pose_RANSAC_mdscale': pose_RANSAC_mdscale,
'pose_RANSAC': pose_RANSAC
}
if dovls:
valnp = val.squeeze().cpu().numpy() == 1
fig, axs = plt.subplots(3, 1, figsize=(16, 9))
rgbvls = tensor2rgb(image1 / 255.0, viewind=0)
axs[2].imshow(tensor2rgb(image2 / 255.0, viewind=0))
for idx, k in enumerate(poses.keys()):
flow_eval = depth2flow(depthgt, depthgt > 0, intrinsic, poses[k])
epe = torch.sum((flowgt_stereo - flow_eval)**2,
dim=1,
keepdim=True).sqrt()
mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt()
out = ((epe[val] > 3.0) & ((epe[val] / mag[val]) > 0.05)).float()
eval_metrics[k][0] += torch.sum(out)
if dovls:
_, _, h, w = depthgt.shape
xx, yy = np.meshgrid(range(w), range(h), indexing='xy')
xxval = xx[valnp]
yyval = yy[valnp]
outnp = out.cpu().numpy() == 1
if k == 'pose_RANSAC':
titlestr = "%s, Performance Level %f, scale_gt: %.4f" % (
k, (torch.sum(out).item() / out.shape[0]), scale)
else:
titlestr = "%s, Performance Level %f, reldiff: %.4f" % (
k, (torch.sum(out).item() / out.shape[0]),
np.abs(scale_md - scale) / scale)
axs[idx].scatter(xxval, yyval, 0.01, 'b')
axs[idx].scatter(xxval[outnp], yyval[outnp], 1, 'r')
axs[idx].imshow(rgbvls)
axs[idx].title.set_text(titlestr)
if dovls:
os.makedirs(args.vlsroot, exist_ok=True)
plt.savefig(os.path.join(args.vlsroot,
"{}.png".format(tagname.split('/')[0])),
bbox_inches='tight',
pad_inches=0)
plt.close()
eval_metrics['pixelnum'][0] += torch.sum(val)
eval_metrics['framenum'][0] += 1
if args.distributed:
for k in eval_metrics.keys():
dist.all_reduce(tensor=eval_metrics[k],
op=dist.ReduceOp.SUM,
group=group)
if args.gpu == 0:
print_keys = ['pose_RANSAC_mdscale', 'pose_RANSAC']
print_str = "Evaluate Poses on Full Split %d samples" % (
eval_metrics['framenum'][0].item())
for key in print_keys:
print_str += ", %s-out: %f" % (key, eval_metrics[key][0].item() /
eval_metrics['pixelnum'][0].item())
print(print_str)
def validate_kitti(model, args, eval_loader, group, isdeepv2d=False):
""" Peform validation using the KITTI-2015 (train) split """
""" Peform validation using the KITTI-2015 (train) split """
model.eval()
gpu = args.gpu
eval_measures_depth = torch.zeros(10).cuda(device=gpu)
vlsroot = '/media/shengjie/disk1/visualization/paper_depthvls'
residual_root = os.path.join(vlsroot, 'residual_vls')
bts_root = os.path.join(vlsroot, 'bts_vls')
ours_root = os.path.join(vlsroot, 'ours_vls')
deepv2d_root = os.path.join(vlsroot, 'deepv2d_vls')
rgb_root = os.path.join(vlsroot, 'rgb_in')
# os.makedirs(residual_root, exist_ok=True)
# os.makedirs(bts_root, exist_ok=True)
# os.makedirs(ours_root, exist_ok=True)
# os.makedirs(deepv2d_root, exist_ok=True)
# os.makedirs(rgb_root, exist_ok=True)
for val_id, data_blob in enumerate(tqdm(eval_loader)):
image1 = data_blob['img1'].cuda(gpu) / 255.0
image2 = data_blob['img2'].cuda(gpu) / 255.0
intrinsic = data_blob['intrinsic'].cuda(gpu)
insmap = data_blob['insmap'].cuda(gpu)
posepred = data_blob['posepred'].cuda(gpu)
depthgt = data_blob['depthmap'].cuda(gpu)
if not args.initbymD:
mD_pred = data_blob['depthpred'].cuda(gpu)
else:
mD_pred = data_blob['mdDepth_pred'].cuda(gpu)
svname = "{}.png".format(str(val_id).zfill(10))
mD_pred_clipped = torch.clamp_min(mD_pred, min=args.min_depth_pred)
outputs = model(image1, image2, mD_pred_clipped, intrinsic, posepred, insmap)
predread = outputs[('depth', 2)]
depthpred_deepv2d = data_blob['depthpred_deepv2d'].cuda(gpu)
sigmoidact = outputs[('residualdepth', 2)]
# tensor2disp(1 / mD_pred_clipped, vmax=0.15, viewind=0).save(os.path.join(bts_root, svname))
# tensor2disp(1 / predread, vmax=0.15, viewind=0).save(os.path.join(ours_root, svname))
# tensor2disp(1 / depthpred_deepv2d, vmax=0.15, viewind=0).save(os.path.join(deepv2d_root, svname))
# tensor2rgb(image1, viewind=0).save(os.path.join(rgb_root, svname))
# tensor2grad(sigmoidact, pos_bar=0.1, neg_bar=-0.1, viewind=0).save(os.path.join(residual_root, svname))
fig1 = tensor2rgb(image1, viewind=0)
fig1_2 = tensor2rgb(image2, viewind=0)
fig2 = tensor2disp(1 / depthpred_deepv2d, vmax=0.15, viewind=0)
fig3 = tensor2disp(1 / mD_pred_clipped, vmax=0.15, viewind=0)
fig4 = tensor2grad(sigmoidact, pos_bar=0.1, neg_bar=-0.1, viewind=0)
fig5 = tensor2disp(1 / predread, vmax=0.15, viewind=0)
figs = concat_imgs([fig1, fig1_2, fig2, fig3, fig4, fig5])
figc1 = np.concatenate([np.array(figs[0]), np.array(figs[1])], axis=0)
figc2 = np.concatenate([np.array(figs[4]), np.array(figs[2])], axis=0)
figc3 = np.concatenate([np.array(figs[3]), np.array(figs[5])], axis=0)
imgvls = np.concatenate([figc1, figc2, figc3], axis=1)
imgvls = Image.fromarray(imgvls)
imgvls.save(os.path.join(vlsroot, svname))
selector = ((depthgt > 0) * (predread > 0) * (depthgt > args.min_depth_eval) * (depthgt < args.max_depth_eval)).float()
predread = torch.clamp(predread, min=args.min_depth_eval, max=args.max_depth_eval)
depth_gt_flatten = depthgt[selector == 1].cpu().numpy()
pred_depth_flatten = predread[selector == 1].cpu().numpy()
pred_depth_flatten = np.median(depth_gt_flatten/pred_depth_flatten) * pred_depth_flatten
eval_measures_depth_np = compute_errors(gt=depth_gt_flatten, pred=pred_depth_flatten)
eval_measures_depth[:9] += torch.tensor(eval_measures_depth_np).cuda(device=gpu)
eval_measures_depth[9] += 1
if args.distributed:
dist.all_reduce(tensor=eval_measures_depth, op=dist.ReduceOp.SUM, group=group)
if args.gpu == 0:
eval_measures_depth[0:9] = eval_measures_depth[0:9] / eval_measures_depth[9]
eval_measures_depth = eval_measures_depth.cpu().numpy()
print('Computing Depth errors for %f eval samples' % (eval_measures_depth[9].item()))
print("{:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}".format('silog', 'abs_rel', 'log10', 'rms', 'sq_rel', 'log_rms', 'd1', 'd2', 'd3'))
for i in range(8):
print('{:7.3f}, '.format(eval_measures_depth[i]), end='')
print('{:7.3f}'.format(eval_measures_depth[8]))
return {'silog': float(eval_measures_depth[0]),
'abs_rel': float(eval_measures_depth[1]),
'log10': float(eval_measures_depth[2]),
'rms': float(eval_measures_depth[3]),
'sq_rel': float(eval_measures_depth[4]),
'log_rms': float(eval_measures_depth[5]),
'd1': float(eval_measures_depth[6]),
'd2': float(eval_measures_depth[7]),
'd3': float(eval_measures_depth[8])
}
else:
return None
def validate_RANSAC_odom_relpose(args, eval_loader, banins=False, bangrad=False, samplenum=50000):
if bangrad:
gradComputer = None
else:
gradComputer = GradComputer()
for val_id, data_blob in enumerate(tqdm(eval_loader)):
insmap = data_blob['insmap']
intrinsic = data_blob['intrinsic']
flowpred = data_blob['flowpred_RAFT']
mdDepth_pred = data_blob['mdDepth']
pose_bs = data_blob['posepred_bs']
tag = data_blob['tag'][0]
if torch.sum(torch.abs(data_blob['img1'] - data_blob['img2'])) < 1:
R = np.eye(3)
t = np.array([[0, 0, -1]]).T
scale = 0
else:
R, t, scale, _ = inf_pose_flow(flowpred, insmap, mdDepth_pred, intrinsic, int(val_id + 10), gradComputer=gradComputer, banins=banins, samplenum=samplenum)
self_pose = np.eye(4)
self_pose[0:3, 0:3] = R
self_pose[0:3, 3:4] = t * scale
pose_bs_np = pose_bs[0].cpu().numpy()
pose_bs_np = pose_bs_np @ np.linalg.inv(pose_bs_np[0]) @ self_pose
pose_bs_np[0] = self_pose
seq, frmidx = tag.split(' ')
exportfold = os.path.join(args.vls_root, seq, 'image_02')
insmap_np = data_blob['insmap'].squeeze().numpy()
xx, yy = np.meshgrid(range(insmap_np.shape[1]), range(insmap_np.shape[0]), indexing='xy')
fig, ax = plt.subplots(figsize=(16,9))
ax.imshow(vls_ins(np.array(tensor2rgb(data_blob['img1'], viewind=0)), insmap_np))
for k in np.unique(insmap_np):
if k == 0:
continue
xxf = xx[insmap_np == k]
yyf = yy[insmap_np == k]
xmin = xxf.min()
xmax = xxf.max()
ymin = yyf.min()
ymax = yyf.max()
if (ymax - ymin) * (xmax - xmin) < 1000:
continue
rect = patches.Rectangle((xmin, ymax), xmax - xmin, ymin - ymax, linewidth=1, facecolor='none', edgecolor='r')
ax.add_patch(rect)
ins_relpose = pose_bs[0, k].cpu().numpy() @ np.linalg.inv(pose_bs[0, 0].cpu().numpy())
mvdist = np.sqrt(np.sum(ins_relpose[0:3, 3:4] ** 2))
mvdist_transed = np.sqrt(np.sum((pose_bs_np[k] @ np.linalg.inv(pose_bs_np[0]))[0:3, 3:4] ** 2))
assert np.abs(mvdist_transed - mvdist) < 1e-3
ax.text(xmin + 5, ymin + 10, '%.3f' % mvdist, fontsize=6, c='r', weight='bold')
plt.axis('off')
os.makedirs(exportfold, exist_ok=True)
plt.savefig(os.path.join(exportfold, "{}.png".format(frmidx)), bbox_inches='tight', pad_inches=0)
plt.close()
def validate_RANSAC_poses(args,
eval_loader,
group,
num_frames=5,
dovls=False,
gtscale=True):
gpu = args.gpu
eval_metrics = {
'deepv2d_pose': torch.zeros(1).cuda(device=gpu),
'pose_RANSAC': torch.zeros(num_frames).cuda(device=gpu),
'pixelnum': torch.zeros(1).cuda(device=gpu),
'framenum': torch.zeros(1).cuda(device=gpu)
}
gradComputer = GradComputer()
for val_id, data_blob in enumerate(tqdm(eval_loader)):
image1 = data_blob['img1'].cuda(gpu)
image2 = data_blob['img2'].cuda(gpu)
insmap = data_blob['insmap'].cuda(gpu)
intrinsic = data_blob['intrinsic'].cuda(gpu)
depthgt = data_blob['depthmap'].cuda(gpu)
valid_flow = data_blob['valid_flow'].cuda(gpu) == 1
tagname = data_blob['tag'][0]
flowgt_stereo = data_blob['flowgt_stereo'].cuda(gpu)
posepred_deepv2d = data_blob['posepred_deepv2d'].cuda(gpu)
flowpred = data_blob['flowpred'].cuda(gpu)
mdDepth_pred = data_blob['mdDepth_pred'].cuda(gpu)
# mdDepth_pred = data_blob['depthpred_deepv2d'].cuda(gpu)
mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt()
val = (mag < MAX_FLOW) * (insmap == 0) * valid_flow * (depthgt > 0)
if not gtscale:
R, t, scale, flow_sel_mag, istatic, insdict = inf_pose_flow(
image1,
image2,
flowpred,
insmap,
mdDepth_pred,
intrinsic,
int(val_id + args.gpu * 100000),
gradComputer=gradComputer)
else:
R, t, scale, flow_sel_mag, istatic, insdict = inf_pose_flow(
image1,
image2,
flowpred,
insmap,
depthgt,
intrinsic,
int(val_id + args.gpu * 100000),
gradComputer=None)
pose_RANSAC = torch.clone(posepred_deepv2d)
if istatic:
pose_RANSAC[0, 0, :, :] = torch.eye(4, device=intrinsic.device)
else:
pose_RANSAC[0, 0, 0:3, 0:3] = torch.from_numpy(R).float().cuda(
intrinsic.device)
pose_RANSAC[0, 0, 0:3, 3:4] = torch.from_numpy(
t * scale).float().cuda(intrinsic.device)
poses = {'deepv2d_pose': posepred_deepv2d, 'pose_RANSAC': pose_RANSAC}
if dovls:
valnp = val.squeeze().cpu().numpy() == 1
fig, axs = plt.subplots(3, 1, figsize=(16, 9))
rgbvls = tensor2rgb(image1 / 255.0, viewind=0)
axs[2].imshow(tensor2rgb(image2 / 255.0, viewind=0))
for idx, k in enumerate(poses.keys()):
flow_eval = depth2flow(depthgt, depthgt > 0, intrinsic, poses[k])
epe = torch.sum((flowgt_stereo - flow_eval)**2,
dim=1,
keepdim=True).sqrt()
mag = torch.sum(flowgt_stereo**2, dim=1, keepdim=True).sqrt()
out = ((epe[val] > 3.0) & ((epe[val] / mag[val]) > 0.05)).float()
eval_metrics[k][0] += torch.sum(out)
if dovls:
_, _, h, w = depthgt.shape
xx, yy = np.meshgrid(range(w), range(h), indexing='xy')
xxval = xx[valnp]
yyval = yy[valnp]
outnp = out.cpu().numpy() == 1
titlestr = "%s, Performance Level %f, flows_mag %f" % (k, (
torch.sum(out).item() / out.shape[0]), flow_sel_mag)
if istatic:
titlestr += ", static"
else:
titlestr += ", moving"
axs[idx].scatter(xxval, yyval, 0.01, 'b')
axs[idx].scatter(xxval[outnp], yyval[outnp], 1, 'r')
if k == 'pose_RANSAC':
insmap_np = insmap.squeeze().cpu().numpy()
for insid in np.unique(insmap_np):
if insid == 0:
continue
if insid not in insdict.keys():
continue
insdata = insdict[insid]
rect = patches.Rectangle(insdata['bbox'][0],
insdata['bbox'][1],
insdata['bbox'][2],
linewidth=1,
edgecolor='r',
facecolor='none')
axs[idx].add_patch(rect)
axs[idx].text(insdata['bbox'][0][0] + 5,
insdata['bbox'][0][1] +
insdata['bbox'][2] + 20,
'%.3f' % (insdata['ave_dist']),
fontsize=10,
c='r',
weight='bold')
axs[idx].imshow(rgbvls)
axs[idx].title.set_text(titlestr)
if dovls:
os.makedirs(args.vlsroot, exist_ok=True)
plt.savefig(os.path.join(args.vlsroot,
"{}.png".format(tagname.split('/')[0])),
bbox_inches='tight',
pad_inches=0)
plt.close()
eval_metrics['pixelnum'][0] += torch.sum(val)
eval_metrics['framenum'][0] += 1
if args.distributed:
for k in eval_metrics.keys():
dist.all_reduce(tensor=eval_metrics[k],
op=dist.ReduceOp.SUM,
group=group)
if args.gpu == 0:
print_keys = ['deepv2d_pose', 'pose_RANSAC']
print_str = "Evaluate Poses on Full Split %d samples" % (
eval_metrics['framenum'][0].item())
for key in print_keys:
print_str += ", %s-out: %f" % (key, eval_metrics[key][0].item() /
eval_metrics['pixelnum'][0].item())
print(print_str)
def validate_RANSAC_odom_relpose(args, eval_loader, banins=False, bangrad=False, samplenum=50000):
if bangrad:
gradComputer = None
else:
gradComputer = GradComputer()
ssim = SSIM()
trialtime = 32
os.makedirs(args.vls_root, exist_ok=True)
for val_id, data_blob in enumerate(tqdm(eval_loader)):
insmap = data_blob['insmap']
intrinsic = data_blob['intrinsic']
flowpred = data_blob['flowpred_RAFT']
mdDepth_pred = data_blob['mdDepth']
tag = data_blob['tag'][0]
flowgt = data_blob['flowgt']
rgb1 = data_blob['img1'] / 255.0
rgb2 = data_blob['img2'] / 255.0
insselector = (insmap == 0).float()
stereogt = data_blob['stereogt']
flowgt_valid = data_blob['flowgt_valid'] * insselector * (stereogt > 0).float()
self_poses = list()
breakflag = False
for k in range(trialtime):
R, t, scale, _ = inf_pose_flow(flowpred, insmap, mdDepth_pred, intrinsic, int(val_id * 500 + k), gradComputer=gradComputer, banins=banins, samplenum=samplenum)
if scale == 0:
breakflag = True
else:
self_pose = np.eye(4)
self_pose[0:3, 0:3] = R
self_pose[0:3, 3:4] = t * scale
self_pose = torch.from_numpy(self_pose).float()
self_poses.append(self_pose)
if breakflag:
continue
rpj_loss_rec = list()
flow_loss_rec = list()
for k in range(trialtime):
self_pose = self_poses[k]
mdDepth_pred_sqz = mdDepth_pred.squeeze()
_, _, h, w = insmap.shape
xx, yy = np.meshgrid(range(w), range(h), indexing='xy')
xxt = torch.from_numpy(xx)
yyt = torch.from_numpy(yy)
ones = torch.ones_like(xxt)
pts3d = torch.stack([xxt * mdDepth_pred_sqz, yyt * mdDepth_pred_sqz, mdDepth_pred_sqz, ones], dim=2).unsqueeze(-1)
intrinsic_sqz = intrinsic.squeeze()
projM = intrinsic_sqz @ self_pose @ torch.inverse(intrinsic_sqz)
projM_ex = projM.unsqueeze(0).unsqueeze(0).expand([h, w, -1, -1])
prj2d = projM_ex @ pts3d
prj2d_x = prj2d[:, :, 0] / prj2d[:, :, 2]
prj2d_y = prj2d[:, :, 1] / prj2d[:, :, 2]
flowx = prj2d_x.squeeze() - xxt
flowy = prj2d_y.squeeze() - yyt
flowpred_depth = torch.stack([flowx, flowy], dim=0).unsqueeze(0)
# Image.fromarray(flow_to_image(flowpred_depth[0].permute([1, 2, 0]).numpy(), rad_max=300)).show()
# Image.fromarray(flow_to_image(flowpred[0].permute([1, 2, 0]).numpy(), rad_max=300)).show()
prj2d_x = (prj2d_x / (w - 1) - 0.5) * 2
prj2d_y = (prj2d_y / (h - 1) - 0.5) * 2
prj2dpx = torch.cat([prj2d_x, prj2d_y], dim=-1).unsqueeze(0)
rgb1_recon = F.grid_sample(rgb2, prj2dpx, mode='bilinear', align_corners=False)
rpj_loss, rpj_loss_imgsize = reprojetion_loss(rgb1, rgb1_recon, ssim, insselector)
mdDepth_pred_sqz = stereogt.squeeze()
_, _, h, w = insmap.shape
xx, yy = np.meshgrid(range(w), range(h), indexing='xy')
xxt = torch.from_numpy(xx)
yyt = torch.from_numpy(yy)
ones = torch.ones_like(xxt)
pts3d = torch.stack([xxt * mdDepth_pred_sqz, yyt * mdDepth_pred_sqz, mdDepth_pred_sqz, ones], dim=2).unsqueeze(-1)
intrinsic_sqz = intrinsic.squeeze()
projM = intrinsic_sqz @ self_pose @ torch.inverse(intrinsic_sqz)
projM_ex = projM.unsqueeze(0).unsqueeze(0).expand([h, w, -1, -1])
prj2d = projM_ex @ pts3d
prj2d_x = prj2d[:, :, 0] / prj2d[:, :, 2]
prj2d_y = prj2d[:, :, 1] / prj2d[:, :, 2]
flowx = prj2d_x.squeeze() - xxt
flowy = prj2d_y.squeeze() - yyt
flowpred_depth = torch.stack([flowx, flowy], dim=0).unsqueeze(0)
flow_loss = torch.sum(torch.abs(flowpred_depth - flowgt), dim=1, keepdim=True)
flow_loss_imgsize = flow_loss * flowgt_valid
flow_loss = torch.sum(flow_loss * flowgt_valid) / torch.sum(flowgt_valid)
# tensor2rgb(rgb1_recon, viewind=0).show()
rpj_loss_rec.append(rpj_loss.item())
flow_loss_rec.append(flow_loss.item())
if k == 0:
fig = plt.figure(figsize=(16, 9))
fig.add_subplot(4, 1, 1)
plt.imshow(tensor2rgb(rgb1, viewind=0))
fig.add_subplot(4, 1, 2)
plt.imshow(tensor2disp(rpj_loss_imgsize, vmax=0.1))
fig.add_subplot(4, 1, 3)
plt.imshow(tensor2disp(flow_loss_imgsize, vmax=3))
fig.add_subplot(4, 1, 4)
plt.scatter(flow_loss_rec, rpj_loss_rec)
plt.xlabel('flow loss')
plt.ylabel('photometric loss')
svpath = os.path.join(args.vls_root, tag.split(' ')[0].split('/')[1] + '.png')
plt.savefig(svpath, bbox_inches='tight')
plt.close()
def validate_RANSAC_odom_relpose(args, eval_loader, samplenum=50000, iters=0):
gradComputer = GradComputer()
# optimized = 0
# skipped = 0
# unresolved = 0
bestids = list()
for val_id, data_blob in enumerate(tqdm(eval_loader)):
intrinsic = data_blob['intrinsic']
flowpred = data_blob['flowpred_RAFT']
mdDepth_pred = data_blob['mdDepth']
# mdDepth_pred = data_blob['gtDepth']
tag = data_blob['tag'][0]
seq, frmidx = tag.split(' ')
exportfold = os.path.join(args.export_root, str(iters).zfill(3), seq)
os.makedirs(exportfold, exist_ok=True)
export_root = os.path.join(exportfold, frmidx.zfill(5) + '.pickle')
if torch.sum(torch.abs(data_blob['img1'] - data_blob['img2'])) < 1:
R = np.eye(3)
t = np.array([[0, 0, -1]]).T
scale = 1e-10
bestid = -1
else:
R, t, scale, pts1_inliers, pts2_inliers, bestid = inf_pose_flow(
flowpred,
mdDepth_pred,
intrinsic,
int(iters * eval_loader.__len__() + val_id),
gradComputer=gradComputer,
samplenum=samplenum,
ban_scale_check=args.ban_scale_check)
self_pose = np.eye(4)
self_pose[0:3, 0:3] = R
self_pose[0:3, 3:4] = t * scale
bestids.append(bestid)
with open(export_root, 'wb') as handle:
pickle.dump(self_pose, handle, protocol=pickle.HIGHEST_PROTOCOL)
# if bestid == -1:
flow_numpy = flowpred[0].cpu().permute([1, 2, 0]).numpy()
_, _, h, w = mdDepth_pred.shape
xx, yy = np.meshgrid(range(w), range(h), indexing='xy')
xxf = xx.flatten()
yyf = yy.flatten()
rndidx = np.random.randint(0, xxf.shape[0], 100)
xxf_rnd = xxf[rndidx]
yyf_rnd = yyf[rndidx]
selector = mdDepth_pred.squeeze().cpu().numpy()[yyf_rnd, xxf_rnd] > 0
xxf_rnd = xxf_rnd[selector]
yyf_rnd = yyf_rnd[selector]
flowx = flow_numpy[yyf_rnd, xxf_rnd, 0]
flowy = flow_numpy[yyf_rnd, xxf_rnd, 1]
depthf = mdDepth_pred.squeeze().cpu().numpy()[yyf_rnd, xxf_rnd]
intrinsic_np = intrinsic.squeeze().numpy()
pts3d = np.stack([
xxf_rnd * depthf, yyf_rnd * depthf,
np.ones_like(yyf_rnd) * depthf,
np.ones_like(yyf_rnd)
],
axis=0)
pts3d = intrinsic_np @ self_pose @ np.linalg.inv(intrinsic_np) @ pts3d
pts3d[0, :] = pts3d[0, :] / pts3d[2, :]
pts3d[1, :] = pts3d[1, :] / pts3d[2, :]
image1 = data_blob['img1']
image2 = data_blob['img2']
vlsrgb1 = tensor2rgb(image1 / 255.0, viewind=0)
vlsrgb2 = tensor2rgb(image2 / 255.0, viewind=0)
fig = plt.figure(figsize=(12, 9))
plt.subplot(2, 1, 1)
plt.scatter(xxf_rnd, yyf_rnd, 5, 'g')
plt.imshow(vlsrgb1)
plt.axis('off')
plt.subplot(2, 1, 2)
plt.scatter(flowx + xxf_rnd, flowy + yyf_rnd, 5, 'r')
plt.scatter(pts3d[0, :], pts3d[1, :], 5, 'b')
plt.axis('off')
# plt.legend(['RAFT Flow', 'Initial Depth + Intial Pose'])
plt.imshow(vlsrgb2)
# plt.show()
plt.savefig(
os.path.join("/media/shengjie/disk1/visualization/nyu_pose_vls",
"{}_{}.jpg".format(seq,
str(frmidx).zfill(5))),
bbox_inches='tight',
pad_inches=0,
)
plt.close()
