def debug(self, batch, output, iter_id):
opt = self.opt
wh = output['wh'] if opt.reg_bbox else None
reg = output['reg'] if opt.reg_offset else None
dets = ddd_decode(output['hm'], output['rot'], output['dep'],
output['dim'], wh=wh, reg=reg, K=opt.K)
# x, y, score, r1-r8, depth, dim1-dim3, cls
dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])
calib = batch['meta']['calib'].detach().numpy()
# x, y, score, rot, depth, dim1, dim2, dim3
# if opt.dataset == 'gta':
# dets[:, 12:15] /= 3
dets_pred = ddd_post_process(
dets.copy(), batch['meta']['c'].detach().numpy(),
batch['meta']['s'].detach().numpy(), calib, opt)
dets_gt = ddd_post_process(
batch['meta']['gt_det'].detach().numpy().copy(),
batch['meta']['c'].detach().numpy(),
batch['meta']['s'].detach().numpy(), calib, opt)
#for i in range(input.size(0)):
for i in range(1):
debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug==3),
theme=opt.debugger_theme)
img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = ((img * self.opt.std + self.opt.mean) * 255.).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'hm_pred')
debugger.add_blend_img(img, gt, 'hm_gt')
# decode
debugger.add_ct_detection(
img, dets[i], show_box=opt.reg_bbox, center_thresh=opt.center_thresh,
img_id='det_pred')
debugger.add_ct_detection(
img, batch['meta']['gt_det'][i].cpu().numpy().copy(),
show_box=opt.reg_bbox, img_id='det_gt')
debugger.add_3d_detection(
batch['meta']['image_path'][i], dets_pred[i], calib[i],
center_thresh=opt.center_thresh, img_id='add_pred')
debugger.add_3d_detection(
batch['meta']['image_path'][i], dets_gt[i], calib[i],
center_thresh=opt.center_thresh, img_id='add_gt')
# debugger.add_bird_view(
# dets_pred[i], center_thresh=opt.center_thresh, img_id='bird_pred')
# debugger.add_bird_view(dets_gt[i], img_id='bird_gt')
debugger.add_bird_views(
dets_pred[i], dets_gt[i],
center_thresh=opt.center_thresh, img_id='bird_pred_gt')
# debugger.add_blend_img(img, pred, 'out', white=True)
debugger.compose_vis_add(
batch['meta']['image_path'][i], dets_pred[i], calib[i],
opt.center_thresh, pred, 'bird_pred_gt', img_id='out')
# debugger.add_img(img, img_id='out')
if opt.debug ==4:
debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id, dataset):
opt = self.opt
if 'pre_hm' in batch:
output.update({'pre_hm': batch['pre_hm']})
dets = fusion_decode(output, K=opt.K, opt=opt)
for k in dets:
dets[k] = dets[k].detach().cpu().numpy()
dets_gt = batch['meta']['gt_det']
for i in range(1):
debugger = Debugger(opt=opt, dataset=dataset)
img = batch['image'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img,
pred,
'pred_hm',
trans=self.opt.hm_transparency)
debugger.add_blend_img(img,
gt,
'gt_hm',
trans=self.opt.hm_transparency)
debugger.add_img(img, img_id='img')
# show point clouds
if opt.pointcloud:
pc_2d = batch['pc_2d'][i].detach().cpu().numpy()
pc_3d = None
pc_N = batch['pc_N'][i].detach().cpu().numpy()
debugger.add_img(img, img_id='pc')
debugger.add_pointcloud(pc_2d, pc_N, img_id='pc')
if 'pc_hm' in opt.pc_feat_lvl:
channel = opt.pc_feat_channels['pc_hm']
pc_hm = debugger.gen_colormap(
batch['pc_hm'][i][channel].unsqueeze(
0).detach().cpu().numpy())
debugger.add_blend_img(img,
pc_hm,
'pc_hm',
trans=self.opt.hm_transparency)
if 'pc_dep' in opt.pc_feat_lvl:
channel = opt.pc_feat_channels['pc_dep']
pc_hm = batch['pc_hm'][i][channel].unsqueeze(
0).detach().cpu().numpy()
pc_dep = debugger.add_overlay_img(img, pc_hm, 'pc_dep')
if 'pre_img' in batch:
pre_img = batch['pre_img'][i].detach().cpu().numpy().transpose(
1, 2, 0)
pre_img = np.clip(
((pre_img * dataset.std + dataset.mean) * 255), 0,
255).astype(np.uint8)
debugger.add_img(pre_img, 'pre_img_pred')
debugger.add_img(pre_img, 'pre_img_gt')
if 'pre_hm' in batch:
pre_hm = debugger.gen_colormap(
batch['pre_hm'][i].detach().cpu().numpy())
debugger.add_blend_img(pre_img,
pre_hm,
'pre_hm',
trans=self.opt.hm_transparency)
debugger.add_img(img, img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_img(img, img_id='out_pred_amodal')
debugger.add_img(img, img_id='out_gt_amodal')
# Predictions
for k in range(len(dets['scores'][i])):
if dets['scores'][i, k] > opt.vis_thresh:
debugger.add_coco_bbox(dets['bboxes'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets['bboxes_amodal'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred_amodal')
if 'hps' in opt.heads and int(dets['clses'][i, k]) == 0:
debugger.add_coco_hp(dets['hps'][i, k] *
opt.down_ratio,
img_id='out_pred')
if 'tracking' in opt.heads:
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='out_pred')
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='pre_img_pred')
# Ground truth
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt['scores'][i])):
if dets_gt['scores'][i][k] > opt.vis_thresh:
if 'dep' in dets_gt.keys():
dist = dets_gt['dep'][i][k]
if len(dist) > 1:
dist = dist[0]
else:
dist = -1
debugger.add_coco_bbox(dets_gt['bboxes'][i][k] *
opt.down_ratio,
dets_gt['clses'][i][k],
dets_gt['scores'][i][k],
img_id='out_gt',
dist=dist)
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets_gt['bboxes_amodal'][i, k] *
opt.down_ratio,
dets_gt['clses'][i, k],
dets_gt['scores'][i, k],
img_id='out_gt_amodal')
if 'hps' in opt.heads and \
(int(dets['clses'][i, k]) == 0):
debugger.add_coco_hp(dets_gt['hps'][i][k] *
opt.down_ratio,
img_id='out_gt')
if 'tracking' in opt.heads:
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='out_gt')
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='pre_img_gt')
if 'hm_hp' in opt.heads:
pred = debugger.gen_colormap_hp(
output['hm_hp'][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(
batch['hm_hp'][i].detach().cpu().numpy())
debugger.add_blend_img(img,
pred,
'pred_hmhp',
trans=self.opt.hm_transparency)
debugger.add_blend_img(img,
gt,
'gt_hmhp',
trans=self.opt.hm_transparency)
if 'rot' in opt.heads and 'dim' in opt.heads and 'dep' in opt.heads:
dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt}
calib = batch['meta']['calib'].detach().numpy() \
if 'calib' in batch['meta'] else None
det_pred = generic_post_process(
opt, dets, batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(), output['hm'].shape[2],
output['hm'].shape[3], self.opt.num_classes, calib)
det_gt = generic_post_process(opt,
dets_gt,
batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(),
output['hm'].shape[2],
output['hm'].shape[3],
self.opt.num_classes,
calib,
is_gt=True)
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_pred[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_pred')
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_gt[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_gt')
pc_3d = None
if opt.pointcloud:
pc_3d = batch['pc_3d'].cpu().numpy()
debugger.add_bird_views(det_pred[i],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id='bird_pred_gt',
pc_3d=pc_3d,
show_velocity=opt.show_velocity)
debugger.add_bird_views([],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id='bird_gt',
pc_3d=pc_3d,
show_velocity=opt.show_velocity)
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id, dataset):
opt = self.opt
if "pre_hm" in batch:
output.update({"pre_hm": batch["pre_hm"]})
dets = generic_decode(output, K=opt.K, opt=opt)
for k in dets:
dets[k] = dets[k].detach().cpu().numpy()
dets_gt = batch["meta"]["gt_det"]
for i in range(1):
debugger = Debugger(opt=opt, dataset=dataset)
img = batch["image"][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * dataset.std + dataset.mean) * 255.0), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output["hm"][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch["hm"][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, "pred_hm")
debugger.add_blend_img(img, gt, "gt_hm")
if "pre_img" in batch:
pre_img = batch["pre_img"][i].detach().cpu().numpy().transpose(
1, 2, 0)
pre_img = np.clip(
((pre_img * dataset.std + dataset.mean) * 255), 0,
255).astype(np.uint8)
debugger.add_img(pre_img, "pre_img_pred")
debugger.add_img(pre_img, "pre_img_gt")
if "pre_hm" in batch:
pre_hm = debugger.gen_colormap(
batch["pre_hm"][i].detach().cpu().numpy())
debugger.add_blend_img(pre_img, pre_hm, "pre_hm")
debugger.add_img(img, img_id="out_pred")
if "ltrb_amodal" in opt.heads:
debugger.add_img(img, img_id="out_pred_amodal")
debugger.add_img(img, img_id="out_gt_amodal")
# Predictions
for k in range(len(dets["scores"][i])):
if dets["scores"][i, k] > opt.vis_thresh:
debugger.add_coco_bbox(
dets["bboxes"][i, k] * opt.down_ratio,
dets["clses"][i, k],
dets["scores"][i, k],
img_id="out_pred",
)
if "ltrb_amodal" in opt.heads:
debugger.add_coco_bbox(
dets["bboxes_amodal"][i, k] * opt.down_ratio,
dets["clses"][i, k],
dets["scores"][i, k],
img_id="out_pred_amodal",
)
if "hps" in opt.heads and int(dets["clses"][i, k]) == 0:
debugger.add_coco_hp(dets["hps"][i, k] *
opt.down_ratio,
img_id="out_pred")
if "tracking" in opt.heads:
debugger.add_arrow(
dets["cts"][i][k] * opt.down_ratio,
dets["tracking"][i][k] * opt.down_ratio,
img_id="out_pred",
)
debugger.add_arrow(
dets["cts"][i][k] * opt.down_ratio,
dets["tracking"][i][k] * opt.down_ratio,
img_id="pre_img_pred",
)
# Ground truth
debugger.add_img(img, img_id="out_gt")
for k in range(len(dets_gt["scores"][i])):
if dets_gt["scores"][i][k] > opt.vis_thresh:
debugger.add_coco_bbox(
dets_gt["bboxes"][i][k] * opt.down_ratio,
dets_gt["clses"][i][k],
dets_gt["scores"][i][k],
img_id="out_gt",
)
if "ltrb_amodal" in opt.heads:
debugger.add_coco_bbox(
dets_gt["bboxes_amodal"][i, k] * opt.down_ratio,
dets_gt["clses"][i, k],
dets_gt["scores"][i, k],
img_id="out_gt_amodal",
)
if "hps" in opt.heads and (int(dets["clses"][i, k]) == 0):
debugger.add_coco_hp(dets_gt["hps"][i][k] *
opt.down_ratio,
img_id="out_gt")
if "tracking" in opt.heads:
debugger.add_arrow(
dets_gt["cts"][i][k] * opt.down_ratio,
dets_gt["tracking"][i][k] * opt.down_ratio,
img_id="out_gt",
)
debugger.add_arrow(
dets_gt["cts"][i][k] * opt.down_ratio,
dets_gt["tracking"][i][k] * opt.down_ratio,
img_id="pre_img_gt",
)
if "hm_hp" in opt.heads:
pred = debugger.gen_colormap_hp(
output["hm_hp"][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(
batch["hm_hp"][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, "pred_hmhp")
debugger.add_blend_img(img, gt, "gt_hmhp")
if "rot" in opt.heads and "dim" in opt.heads and "dep" in opt.heads:
dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt}
calib = (batch["meta"]["calib"].detach().numpy()
if "calib" in batch["meta"] else None)
det_pred = generic_post_process(
opt,
dets,
batch["meta"]["c"].cpu().numpy(),
batch["meta"]["s"].cpu().numpy(),
output["hm"].shape[2],
output["hm"].shape[3],
self.opt.num_classes,
calib,
)
det_gt = generic_post_process(
opt,
dets_gt,
batch["meta"]["c"].cpu().numpy(),
batch["meta"]["s"].cpu().numpy(),
output["hm"].shape[2],
output["hm"].shape[3],
self.opt.num_classes,
calib,
)
debugger.add_3d_detection(
batch["meta"]["img_path"][i],
batch["meta"]["flipped"][i],
det_pred[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id="add_pred",
)
debugger.add_3d_detection(
batch["meta"]["img_path"][i],
batch["meta"]["flipped"][i],
det_gt[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id="add_gt",
)
debugger.add_bird_views(
det_pred[i],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id="bird_pred_gt",
)
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix="{}".format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id, dataset):
opt = self.opt
if 'pre_hm' in batch:
output.update({'pre_hm': batch['pre_hm']})
dets = generic_decode(output, K=opt.K, opt=opt)
for k in dets:
dets[k] = dets[k].detach().cpu().numpy()
dets_gt = batch['meta']['gt_det']
for i in range(1):
debugger = Debugger(opt=opt, dataset=dataset)
img = batch['image'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hm')
debugger.add_blend_img(img, gt, 'gt_hm')
if 'pre_img' in batch:
pre_img = batch['pre_img'][i].detach().cpu().numpy().transpose(
1, 2, 0)
pre_img = np.clip(
((pre_img * dataset.std + dataset.mean) * 255), 0,
255).astype(np.uint8)
debugger.add_img(pre_img, 'pre_img_pred')
debugger.add_img(pre_img, 'pre_img_gt')
if 'pre_hm' in batch:
pre_hm = debugger.gen_colormap(
batch['pre_hm'][i].detach().cpu().numpy())
debugger.add_blend_img(pre_img, pre_hm, 'pre_hm')
debugger.add_img(img, img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_img(img, img_id='out_pred_amodal')
debugger.add_img(img, img_id='out_gt_amodal')
# Predictions
for k in range(len(dets['scores'][i])):
if dets['scores'][i, k] > opt.vis_thresh:
debugger.add_coco_bbox(dets['bboxes'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets['bboxes_amodal'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred_amodal')
if 'hps' in opt.heads and int(dets['clses'][i, k]) == 0:
debugger.add_coco_hp(dets['hps'][i, k] *
opt.down_ratio,
img_id='out_pred')
if 'tracking' in opt.heads:
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='out_pred')
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='pre_img_pred')
# Ground truth
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt['scores'][i])):
if dets_gt['scores'][i][k] > opt.vis_thresh:
debugger.add_coco_bbox(dets_gt['bboxes'][i][k] *
opt.down_ratio,
dets_gt['clses'][i][k],
dets_gt['scores'][i][k],
img_id='out_gt')
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets_gt['bboxes_amodal'][i, k] *
opt.down_ratio,
dets_gt['clses'][i, k],
dets_gt['scores'][i, k],
img_id='out_gt_amodal')
if 'hps' in opt.heads and \
(int(dets['clses'][i, k]) == 0):
debugger.add_coco_hp(dets_gt['hps'][i][k] *
opt.down_ratio,
img_id='out_gt')
if 'tracking' in opt.heads:
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='out_gt')
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='pre_img_gt')
if 'hm_hp' in opt.heads:
pred = debugger.gen_colormap_hp(
output['hm_hp'][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(
batch['hm_hp'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hmhp')
debugger.add_blend_img(img, gt, 'gt_hmhp')
if 'rot' in opt.heads and 'dim' in opt.heads and 'dep' in opt.heads:
dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt}
calib = batch['meta']['calib'].detach().numpy() \
if 'calib' in batch['meta'] else None
det_pred = generic_post_process(
opt, dets, batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(), output['hm'].shape[2],
output['hm'].shape[3], self.opt.num_classes, calib)
det_gt = generic_post_process(opt, dets_gt,
batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(),
output['hm'].shape[2],
output['hm'].shape[3],
self.opt.num_classes, calib)
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_pred[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_pred')
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_gt[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_gt')
debugger.add_bird_views(det_pred[i],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id='bird_pred_gt')
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
