def debug(self, batch, output, iter_id):
opt = self.opt
reg = output['reg'] if opt.reg_offset else None
hm_hp = output['hm_hp'] if opt.hm_hp else None
hp_offset = output['hp_offset'] if opt.reg_hp_offset else None
dets = multi_pose_decode(
output['hm'], output['wh'], output['hps'],
reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=opt.K)
dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])
dets[:, :, :4] *= opt.input_res / opt.output_res
dets[:, :, 5:39] *= opt.input_res / opt.output_res
dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2])
dets_gt[:, :, :4] *= opt.input_res / opt.output_res
dets_gt[:, :, 5:39] *= opt.input_res / opt.output_res
for i in range(1):
debugger = Debugger(
dataset=opt.dataset, ipynb=(opt.debug==3), theme=opt.debugger_theme)
img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((
img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8)
pred = debugger.gen_colormap(output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hm')
debugger.add_blend_img(img, gt, 'gt_hm')
debugger.add_img(img, img_id='out_pred')
for k in range(len(dets[i])):
if dets[i, k, 4] > opt.center_thresh:
debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1],
dets[i, k, 4], img_id='out_pred')
debugger.add_coco_hp(dets[i, k, 5:39], img_id='out_pred')
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt[i])):
if dets_gt[i, k, 4] > opt.center_thresh:
debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1],
dets_gt[i, k, 4], img_id='out_gt')
debugger.add_coco_hp(dets_gt[i, k, 5:39], img_id='out_gt')
if opt.hm_hp:
pred = debugger.gen_colormap_hp(output['hm_hp'][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(batch['hm_hp'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hmhp')
debugger.add_blend_img(img, gt, 'gt_hmhp')
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id):
cfg = self.cfg
reg = output[3] if cfg.LOSS.REG_OFFSET else None
hm_hp = output[4] if cfg.LOSS.HM_HP else None
hp_offset = output[5] if cfg.LOSS.REG_HP_OFFSET else None
dets = multi_pose_decode(
output[0], output[1], output[2],
reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=cfg.TEST.TOPK)
dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])
dets[:, :, :4] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES
dets[:, :, 5:39] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES
dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2])
dets_gt[:, :, :4] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES
dets_gt[:, :, 5:39] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES
for i in range(1):
debugger = Debugger(
dataset=cfg.SAMPLE_METHOD, ipynb=(cfg.DEBUG==3), theme=cfg.DEBUG_THEME)
img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((
img * np.array(cfg.DATASET.STD).reshape(1,1,3).astype(np.float32) + cfg.DATASET.MEAN) * 255.), 0, 255).astype(np.uint8)
pred = debugger.gen_colormap(output[0][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hm')
debugger.add_blend_img(img, gt, 'gt_hm')
debugger.add_img(img, img_id='out_pred')
for k in range(len(dets[i])):
if dets[i, k, 4] > cfg.MODEL.CENTER_THRESH:
debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1],
dets[i, k, 4], img_id='out_pred')
debugger.add_coco_hp(dets[i, k, 5:39], img_id='out_pred')
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt[i])):
if dets_gt[i, k, 4] > cfg.MODEL.CENTER_THRESH:
debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1],
dets_gt[i, k, 4], img_id='out_gt')
debugger.add_coco_hp(dets_gt[i, k, 5:39], img_id='out_gt')
if cfg.LOSS.HM_HP:
pred = debugger.gen_colormap_hp(output[4][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(batch['hm_hp'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hmhp')
debugger.add_blend_img(img, gt, 'gt_hmhp')
if cfg.DEBUG == 4:
debugger.save_all_imgs(cfg.LOG_DIR, prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id, dataset):
opt = self.opt
if "pre_hm" in batch:
output.update({"pre_hm": batch["pre_hm"]})
dets = generic_decode(output, K=opt.K, opt=opt)
for k in dets:
dets[k] = dets[k].detach().cpu().numpy()
dets_gt = batch["meta"]["gt_det"]
for i in range(1):
debugger = Debugger(opt=opt, dataset=dataset)
img = batch["image"][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * dataset.std + dataset.mean) * 255.0), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output["hm"][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch["hm"][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, "pred_hm")
debugger.add_blend_img(img, gt, "gt_hm")
if "pre_img" in batch:
pre_img = batch["pre_img"][i].detach().cpu().numpy().transpose(
1, 2, 0)
pre_img = np.clip(
((pre_img * dataset.std + dataset.mean) * 255), 0,
255).astype(np.uint8)
debugger.add_img(pre_img, "pre_img_pred")
debugger.add_img(pre_img, "pre_img_gt")
if "pre_hm" in batch:
pre_hm = debugger.gen_colormap(
batch["pre_hm"][i].detach().cpu().numpy())
debugger.add_blend_img(pre_img, pre_hm, "pre_hm")
debugger.add_img(img, img_id="out_pred")
if "ltrb_amodal" in opt.heads:
debugger.add_img(img, img_id="out_pred_amodal")
debugger.add_img(img, img_id="out_gt_amodal")
# Predictions
for k in range(len(dets["scores"][i])):
if dets["scores"][i, k] > opt.vis_thresh:
debugger.add_coco_bbox(
dets["bboxes"][i, k] * opt.down_ratio,
dets["clses"][i, k],
dets["scores"][i, k],
img_id="out_pred",
)
if "ltrb_amodal" in opt.heads:
debugger.add_coco_bbox(
dets["bboxes_amodal"][i, k] * opt.down_ratio,
dets["clses"][i, k],
dets["scores"][i, k],
img_id="out_pred_amodal",
)
if "hps" in opt.heads and int(dets["clses"][i, k]) == 0:
debugger.add_coco_hp(dets["hps"][i, k] *
opt.down_ratio,
img_id="out_pred")
if "tracking" in opt.heads:
debugger.add_arrow(
dets["cts"][i][k] * opt.down_ratio,
dets["tracking"][i][k] * opt.down_ratio,
img_id="out_pred",
)
debugger.add_arrow(
dets["cts"][i][k] * opt.down_ratio,
dets["tracking"][i][k] * opt.down_ratio,
img_id="pre_img_pred",
)
# Ground truth
debugger.add_img(img, img_id="out_gt")
for k in range(len(dets_gt["scores"][i])):
if dets_gt["scores"][i][k] > opt.vis_thresh:
debugger.add_coco_bbox(
dets_gt["bboxes"][i][k] * opt.down_ratio,
dets_gt["clses"][i][k],
dets_gt["scores"][i][k],
img_id="out_gt",
)
if "ltrb_amodal" in opt.heads:
debugger.add_coco_bbox(
dets_gt["bboxes_amodal"][i, k] * opt.down_ratio,
dets_gt["clses"][i, k],
dets_gt["scores"][i, k],
img_id="out_gt_amodal",
)
if "hps" in opt.heads and (int(dets["clses"][i, k]) == 0):
debugger.add_coco_hp(dets_gt["hps"][i][k] *
opt.down_ratio,
img_id="out_gt")
if "tracking" in opt.heads:
debugger.add_arrow(
dets_gt["cts"][i][k] * opt.down_ratio,
dets_gt["tracking"][i][k] * opt.down_ratio,
img_id="out_gt",
)
debugger.add_arrow(
dets_gt["cts"][i][k] * opt.down_ratio,
dets_gt["tracking"][i][k] * opt.down_ratio,
img_id="pre_img_gt",
)
if "hm_hp" in opt.heads:
pred = debugger.gen_colormap_hp(
output["hm_hp"][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(
batch["hm_hp"][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, "pred_hmhp")
debugger.add_blend_img(img, gt, "gt_hmhp")
if "rot" in opt.heads and "dim" in opt.heads and "dep" in opt.heads:
dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt}
calib = (batch["meta"]["calib"].detach().numpy()
if "calib" in batch["meta"] else None)
det_pred = generic_post_process(
opt,
dets,
batch["meta"]["c"].cpu().numpy(),
batch["meta"]["s"].cpu().numpy(),
output["hm"].shape[2],
output["hm"].shape[3],
self.opt.num_classes,
calib,
)
det_gt = generic_post_process(
opt,
dets_gt,
batch["meta"]["c"].cpu().numpy(),
batch["meta"]["s"].cpu().numpy(),
output["hm"].shape[2],
output["hm"].shape[3],
self.opt.num_classes,
calib,
)
debugger.add_3d_detection(
batch["meta"]["img_path"][i],
batch["meta"]["flipped"][i],
det_pred[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id="add_pred",
)
debugger.add_3d_detection(
batch["meta"]["img_path"][i],
batch["meta"]["flipped"][i],
det_gt[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id="add_gt",
)
debugger.add_bird_views(
det_pred[i],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id="bird_pred_gt",
)
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix="{}".format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id, dataset):
opt = self.opt
if 'pre_hm' in batch:
output.update({'pre_hm': batch['pre_hm']})
dets = fusion_decode(output, K=opt.K, opt=opt)
for k in dets:
dets[k] = dets[k].detach().cpu().numpy()
dets_gt = batch['meta']['gt_det']
for i in range(1):
debugger = Debugger(opt=opt, dataset=dataset)
img = batch['image'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img,
pred,
'pred_hm',
trans=self.opt.hm_transparency)
debugger.add_blend_img(img,
gt,
'gt_hm',
trans=self.opt.hm_transparency)
debugger.add_img(img, img_id='img')
# show point clouds
if opt.pointcloud:
pc_2d = batch['pc_2d'][i].detach().cpu().numpy()
pc_3d = None
pc_N = batch['pc_N'][i].detach().cpu().numpy()
debugger.add_img(img, img_id='pc')
debugger.add_pointcloud(pc_2d, pc_N, img_id='pc')
if 'pc_hm' in opt.pc_feat_lvl:
channel = opt.pc_feat_channels['pc_hm']
pc_hm = debugger.gen_colormap(
batch['pc_hm'][i][channel].unsqueeze(
0).detach().cpu().numpy())
debugger.add_blend_img(img,
pc_hm,
'pc_hm',
trans=self.opt.hm_transparency)
if 'pc_dep' in opt.pc_feat_lvl:
channel = opt.pc_feat_channels['pc_dep']
pc_hm = batch['pc_hm'][i][channel].unsqueeze(
0).detach().cpu().numpy()
pc_dep = debugger.add_overlay_img(img, pc_hm, 'pc_dep')
if 'pre_img' in batch:
pre_img = batch['pre_img'][i].detach().cpu().numpy().transpose(
1, 2, 0)
pre_img = np.clip(
((pre_img * dataset.std + dataset.mean) * 255), 0,
255).astype(np.uint8)
debugger.add_img(pre_img, 'pre_img_pred')
debugger.add_img(pre_img, 'pre_img_gt')
if 'pre_hm' in batch:
pre_hm = debugger.gen_colormap(
batch['pre_hm'][i].detach().cpu().numpy())
debugger.add_blend_img(pre_img,
pre_hm,
'pre_hm',
trans=self.opt.hm_transparency)
debugger.add_img(img, img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_img(img, img_id='out_pred_amodal')
debugger.add_img(img, img_id='out_gt_amodal')
# Predictions
for k in range(len(dets['scores'][i])):
if dets['scores'][i, k] > opt.vis_thresh:
debugger.add_coco_bbox(dets['bboxes'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets['bboxes_amodal'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred_amodal')
if 'hps' in opt.heads and int(dets['clses'][i, k]) == 0:
debugger.add_coco_hp(dets['hps'][i, k] *
opt.down_ratio,
img_id='out_pred')
if 'tracking' in opt.heads:
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='out_pred')
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='pre_img_pred')
# Ground truth
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt['scores'][i])):
if dets_gt['scores'][i][k] > opt.vis_thresh:
if 'dep' in dets_gt.keys():
dist = dets_gt['dep'][i][k]
if len(dist) > 1:
dist = dist[0]
else:
dist = -1
debugger.add_coco_bbox(dets_gt['bboxes'][i][k] *
opt.down_ratio,
dets_gt['clses'][i][k],
dets_gt['scores'][i][k],
img_id='out_gt',
dist=dist)
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets_gt['bboxes_amodal'][i, k] *
opt.down_ratio,
dets_gt['clses'][i, k],
dets_gt['scores'][i, k],
img_id='out_gt_amodal')
if 'hps' in opt.heads and \
(int(dets['clses'][i, k]) == 0):
debugger.add_coco_hp(dets_gt['hps'][i][k] *
opt.down_ratio,
img_id='out_gt')
if 'tracking' in opt.heads:
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='out_gt')
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='pre_img_gt')
if 'hm_hp' in opt.heads:
pred = debugger.gen_colormap_hp(
output['hm_hp'][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(
batch['hm_hp'][i].detach().cpu().numpy())
debugger.add_blend_img(img,
pred,
'pred_hmhp',
trans=self.opt.hm_transparency)
debugger.add_blend_img(img,
gt,
'gt_hmhp',
trans=self.opt.hm_transparency)
if 'rot' in opt.heads and 'dim' in opt.heads and 'dep' in opt.heads:
dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt}
calib = batch['meta']['calib'].detach().numpy() \
if 'calib' in batch['meta'] else None
det_pred = generic_post_process(
opt, dets, batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(), output['hm'].shape[2],
output['hm'].shape[3], self.opt.num_classes, calib)
det_gt = generic_post_process(opt,
dets_gt,
batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(),
output['hm'].shape[2],
output['hm'].shape[3],
self.opt.num_classes,
calib,
is_gt=True)
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_pred[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_pred')
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_gt[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_gt')
pc_3d = None
if opt.pointcloud:
pc_3d = batch['pc_3d'].cpu().numpy()
debugger.add_bird_views(det_pred[i],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id='bird_pred_gt',
pc_3d=pc_3d,
show_velocity=opt.show_velocity)
debugger.add_bird_views([],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id='bird_gt',
pc_3d=pc_3d,
show_velocity=opt.show_velocity)
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id, dataset):
opt = self.opt
if 'pre_hm' in batch:
output.update({'pre_hm': batch['pre_hm']})
dets = generic_decode(output, K=opt.K, opt=opt)
for k in dets:
dets[k] = dets[k].detach().cpu().numpy()
dets_gt = batch['meta']['gt_det']
for i in range(1):
debugger = Debugger(opt=opt, dataset=dataset)
img = batch['image'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hm')
debugger.add_blend_img(img, gt, 'gt_hm')
if 'pre_img' in batch:
pre_img = batch['pre_img'][i].detach().cpu().numpy().transpose(
1, 2, 0)
pre_img = np.clip(
((pre_img * dataset.std + dataset.mean) * 255), 0,
255).astype(np.uint8)
debugger.add_img(pre_img, 'pre_img_pred')
debugger.add_img(pre_img, 'pre_img_gt')
if 'pre_hm' in batch:
pre_hm = debugger.gen_colormap(
batch['pre_hm'][i].detach().cpu().numpy())
debugger.add_blend_img(pre_img, pre_hm, 'pre_hm')
debugger.add_img(img, img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_img(img, img_id='out_pred_amodal')
debugger.add_img(img, img_id='out_gt_amodal')
# Predictions
for k in range(len(dets['scores'][i])):
if dets['scores'][i, k] > opt.vis_thresh:
debugger.add_coco_bbox(dets['bboxes'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred')
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets['bboxes_amodal'][i, k] *
opt.down_ratio,
dets['clses'][i, k],
dets['scores'][i, k],
img_id='out_pred_amodal')
if 'hps' in opt.heads and int(dets['clses'][i, k]) == 0:
debugger.add_coco_hp(dets['hps'][i, k] *
opt.down_ratio,
img_id='out_pred')
if 'tracking' in opt.heads:
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='out_pred')
debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio,
dets['tracking'][i][k] *
opt.down_ratio,
img_id='pre_img_pred')
# Ground truth
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt['scores'][i])):
if dets_gt['scores'][i][k] > opt.vis_thresh:
debugger.add_coco_bbox(dets_gt['bboxes'][i][k] *
opt.down_ratio,
dets_gt['clses'][i][k],
dets_gt['scores'][i][k],
img_id='out_gt')
if 'ltrb_amodal' in opt.heads:
debugger.add_coco_bbox(dets_gt['bboxes_amodal'][i, k] *
opt.down_ratio,
dets_gt['clses'][i, k],
dets_gt['scores'][i, k],
img_id='out_gt_amodal')
if 'hps' in opt.heads and \
(int(dets['clses'][i, k]) == 0):
debugger.add_coco_hp(dets_gt['hps'][i][k] *
opt.down_ratio,
img_id='out_gt')
if 'tracking' in opt.heads:
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='out_gt')
debugger.add_arrow(
dets_gt['cts'][i][k] * opt.down_ratio,
dets_gt['tracking'][i][k] * opt.down_ratio,
img_id='pre_img_gt')
if 'hm_hp' in opt.heads:
pred = debugger.gen_colormap_hp(
output['hm_hp'][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(
batch['hm_hp'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hmhp')
debugger.add_blend_img(img, gt, 'gt_hmhp')
if 'rot' in opt.heads and 'dim' in opt.heads and 'dep' in opt.heads:
dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt}
calib = batch['meta']['calib'].detach().numpy() \
if 'calib' in batch['meta'] else None
det_pred = generic_post_process(
opt, dets, batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(), output['hm'].shape[2],
output['hm'].shape[3], self.opt.num_classes, calib)
det_gt = generic_post_process(opt, dets_gt,
batch['meta']['c'].cpu().numpy(),
batch['meta']['s'].cpu().numpy(),
output['hm'].shape[2],
output['hm'].shape[3],
self.opt.num_classes, calib)
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_pred[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_pred')
debugger.add_3d_detection(batch['meta']['img_path'][i],
batch['meta']['flipped'][i],
det_gt[i],
calib[i],
vis_thresh=opt.vis_thresh,
img_id='add_gt')
debugger.add_bird_views(det_pred[i],
det_gt[i],
vis_thresh=opt.vis_thresh,
img_id='bird_pred_gt')
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def test_loader(cfg):
debugger = Debugger((cfg.DEBUG == 3),
theme=cfg.DEBUG_THEME,
num_classes=cfg.MODEL.NUM_CLASSES,
dataset=cfg.SAMPLE_METHOD,
down_ratio=cfg.MODEL.DOWN_RATIO)
Dataset = get_dataset(cfg.SAMPLE_METHOD, cfg.TASK)
val_dataset = Dataset(cfg, 'val')
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
for i, batch_data in enumerate(val_loader):
input_image = batch_data['input']
heat = batch_data['hm']
reg = batch_data['reg']
reg_mask = batch_data['reg_mask']
ind = batch_data['ind']
wh = batch_data['wh']
kps = batch_data['hps']
hps_mask = batch_data['hps_mask']
seg_feat = batch_data['seg']
hm_hp = batch_data['hm_hp']
hp_offset = batch_data['hp_offset']
hp_ind = batch_data['hp_ind']
hp_mask = batch_data['hp_mask']
meta = batch_data['meta']
for k, v in batch_data.items():
if type(v) == type(dict()):
for k1, v1 in v.items():
print(k1)
print(v1)
else:
print(k)
print(v.shape)
print(input_image.shape)
print(hm_hp.shape)
#handle image
input_image = input_image[0].numpy().transpose(1, 2, 0)
input_image = (input_image * STD) + MEAN
input_image = input_image * 255
input_image = input_image.astype(np.uint8)
heat = heat.sigmoid_()
hm_hp = hm_hp.sigmoid_()
num_joints = 17
K = cfg.TEST.TOPK
# perform nms on heatmaps
batch, cat, height, width = heat.size()
heat = _nms(heat)
scores, inds, clses, ys, xs = _topk(heat, K=K)
kps = kps.view(batch, K, num_joints * 2)
kps[..., ::2] += xs.view(batch, K, 1).expand(batch, K, num_joints)
kps[..., 1::2] += ys.view(batch, K, 1).expand(batch, K, num_joints)
xs = xs.view(batch, K, 1) + reg[:, :, 0:1]
ys = ys.view(batch, K, 1) + reg[:, :, 1:2]
wh = wh.view(batch, K, 2)
#weight = _transpose_and_gather_feat(seg, inds)
## you can write (if weight.size(1)!=seg_feat.size(1): 3x3conv else 1x1conv ) here to select seg conv.
## for 3x3
#weight = weight.view([weight.size(1), -1, 3, 3])
pred_seg = seg_feat
clses = clses.view(batch, K, 1).float()
scores = scores.view(batch, K, 1)
bboxes = torch.cat([
xs - wh[..., 0:1] / 2, ys - wh[..., 1:2] / 2,
xs + wh[..., 0:1] / 2, ys + wh[..., 1:2] / 2
],
dim=2)
if hm_hp is not None:
hm_hp = _nms(hm_hp)
thresh = 0.1
kps = kps.view(batch, K, num_joints,
2).permute(0, 2, 1, 3).contiguous() # b x J x K x 2
reg_kps = kps.unsqueeze(3).expand(batch, num_joints, K, K, 2)
hm_score, hm_inds, hm_ys, hm_xs = _topk_channel(hm_hp,
K=K) # b x J x K
hp_offset = hp_offset.view(batch, num_joints, K, 2)
hm_xs = hm_xs + hp_offset[:, :, :, 0]
hm_ys = hm_ys + hp_offset[:, :, :, 1]
mask = (hm_score > thresh).float()
hm_score = (1 - mask) * -1 + mask * hm_score
hm_ys = (1 - mask) * (-10000) + mask * hm_ys
hm_xs = (1 - mask) * (-10000) + mask * hm_xs
hm_kps = torch.stack([hm_xs, hm_ys], dim=-1).unsqueeze(2).expand(
batch, num_joints, K, K, 2)
dist = (((reg_kps - hm_kps)**2).sum(dim=4)**0.5)
min_dist, min_ind = dist.min(dim=3) # b x J x K
hm_score = hm_score.gather(2,
min_ind).unsqueeze(-1) # b x J x K x 1
min_dist = min_dist.unsqueeze(-1)
min_ind = min_ind.view(batch, num_joints, K, 1,
1).expand(batch, num_joints, K, 1, 2)
hm_kps = hm_kps.gather(3, min_ind)
hm_kps = hm_kps.view(batch, num_joints, K, 2)
l = bboxes[:, :, 0].view(batch, 1, K,
1).expand(batch, num_joints, K, 1)
t = bboxes[:, :, 1].view(batch, 1, K,
1).expand(batch, num_joints, K, 1)
r = bboxes[:, :, 2].view(batch, 1, K,
1).expand(batch, num_joints, K, 1)
b = bboxes[:, :, 3].view(batch, 1, K,
1).expand(batch, num_joints, K, 1)
mask = (hm_kps[..., 0:1] < l) + (hm_kps[..., 0:1] > r) + \
(hm_kps[..., 1:2] < t) + (hm_kps[..., 1:2] > b) + \
(hm_score < thresh) + (min_dist > (torch.max(b - t, r - l) * 0.3))
mask = (mask > 0).float().expand(batch, num_joints, K, 2)
kps = (1 - mask) * hm_kps + mask * kps
kps = kps.permute(0, 2, 1,
3).contiguous().view(batch, K, num_joints * 2)
dets = torch.cat([
bboxes, scores, kps,
torch.transpose(hm_score.squeeze(dim=3), 1, 2)
],
dim=2)
dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])
dets, inds = whole_body_post_process(dets.copy(), [meta['c'].numpy()],
[meta['s'].numpy()], 128, 128, 1)
for j in range(1, cfg.MODEL.NUM_CLASSES + 1):
dets[0][j] = np.array(dets[0][j], dtype=np.float32).reshape(-1, 56)
dets[0][j][:, :4] /= 1.
dets[0][j][:, 5:39] /= 1.
print(pred_seg.shape)
seg = pred_seg[0]
trans = get_affine_transform(meta['c'],
meta['s'],
0,
(meta['out_width'], meta['out_height']),
inv=1)
debugger.add_img(image, img_id='multi_pose')
for j in range(1, self.num_classes + 1):
for b_id, detection in enumerate(results[j]):
bbox = detection[:4]
bbox_prob = detection[4]
keypoints = detection[5:39]
keypoints_prob = detection[39:]
if bbox_prob > self.cfg.TEST.VIS_THRESH:
debugger.add_coco_bbox(bbox,
0,
bbox_prob,
img_id='multi_pose')
segment = seg[b_id].detach().cpu().numpy()
segment = cv2.warpAffine(segment,
trans,
(image.shape[1], image.shape[0]),
flags=cv2.INTER_CUBIC)
w, h = bbox[2:4] - bbox[:2]
ct = np.array([(bbox[0] + bbox[2]) / 2,
(bbox[1] + bbox[3]) / 2],
dtype=np.float32)
segment_mask = np.zeros_like(segment)
pad_rate = 0.3
x, y = np.clip([ct[0] - (1 + pad_rate) * w / 2, ct[0] + (1 + pad_rate) * w / 2], 0,
segment.shape[1] - 1).astype(np.int), \
np.clip([ct[1] - (1 + pad_rate) * h / 2, ct[1] + (1 + pad_rate) * h / 2], 0,
segment.shape[0] - 1).astype(np.int)
segment_mask[y[0]:y[1], x[0]:x[1]] = 1
segment = segment_mask * segment
debugger.add_coco_seg(segment, img_id='multi_pose')
debugger.add_coco_hp(keypoints,
keypoints_prob,
img_id='multi_pose')
debugger.show_all_imgs(pause=self.pause)
save_path = os.path.join(SAVE_DIR, '{}.png'.format(i))
cv2.imwrite(save_path, input_image)
def run(self, image_or_path_or_tensor, meta=None):
load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0
merge_time, tot_time = 0, 0
debugger = Debugger(dataset=self.opt.dataset, ipynb=(self.opt.debug==3),
theme=self.opt.debugger_theme)
start_time = time.time()
pre_processed = False
if isinstance(image_or_path_or_tensor, np.ndarray):
image = image_or_path_or_tensor
elif type(image_or_path_or_tensor) == type (''):
image = cv2.imread(image_or_path_or_tensor)
else:
image = image_or_path_or_tensor['image'][0].numpy()
pre_processed_images = image_or_path_or_tensor
pre_processed = True
loaded_time = time.time()
load_time += (loaded_time - start_time)
detections = []
for scale in self.scales:
scale_start_time = time.time()
if not pre_processed:
images, meta = self.pre_process(image, scale, meta)
else:
# import pdb; pdb.set_trace()
images = pre_processed_images['images'][scale][0]
meta = pre_processed_images['meta'][scale]
meta = {k: v.numpy()[0] for k, v in meta.items()}
images = images.to(self.opt.device)
torch.cuda.synchronize()
pre_process_time = time.time()
pre_time += pre_process_time - scale_start_time
output, dets, forward_time = self.process(images, return_time=True)
torch.cuda.synchronize()
net_time += forward_time - pre_process_time
decode_time = time.time()
dec_time += decode_time - forward_time
if self.opt.debug >= 2:
self.debug(debugger, images, dets, output, scale)
dets = self.post_process(dets, meta, scale)
torch.cuda.synchronize()
post_process_time = time.time()
post_time += post_process_time - decode_time
detections.append(dets)
results = self.merge_outputs(detections)
torch.cuda.synchronize()
end_time = time.time()
merge_time += end_time - post_process_time
tot_time += end_time - start_time
if self.opt.debug >= 1:
# print('--->>> base_detector run show_results')
# img_ = self.show_results(debugger, image, results)
debugger.add_img(image, img_id='multi_pose')
for bbox in results[1]:
if bbox[4] > self.opt.vis_thresh:
# 绘制目标物体
# print('------------------>>>add_coco_bbox')
debugger.add_coco_bbox(bbox[:4], 0, bbox[4], img_id='multi_pose')
debugger.add_coco_hp(bbox[5:39], img_id='multi_pose')
# print('-------->>> show_results debugger')
img_ = debugger.show_all_imgs(pause=self.pause)
return img_,{'results': results, 'tot': tot_time, 'load': load_time,
'pre': pre_time, 'net': net_time, 'dec': dec_time,
'post': post_time, 'merge': merge_time}
def run(self, image_or_path_or_tensor, meta=None):
load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0
merge_time, tot_time = 0, 0
debugger = Debugger(dataset=self.opt.dataset,
ipynb=(self.opt.debug == 3),
theme=self.opt.debugger_theme)
start_time = time.time()
pre_processed = False
if isinstance(image_or_path_or_tensor, np.ndarray):
image = image_or_path_or_tensor
elif type(image_or_path_or_tensor) == type(''):
image = cv2.imread(image_or_path_or_tensor)
else:
image = image_or_path_or_tensor['image'][0].numpy()
pre_processed_images = image_or_path_or_tensor
pre_processed = True
loaded_time = time.time()
load_time += (loaded_time - start_time)
detections = []
for scale in self.scales:
scale_start_time = time.time()
if not pre_processed:
images, meta = self.pre_process(image, scale, meta)
else:
# import pdb; pdb.set_trace()
images = pre_processed_images['images'][scale][0]
meta = pre_processed_images['meta'][scale]
meta = {k: v.numpy()[0] for k, v in meta.items()}
images = images.to(self.opt.device)
torch.cuda.synchronize()
pre_process_time = time.time()
pre_time += pre_process_time - scale_start_time
output, dets, forward_time = self.process(images, return_time=True)
torch.cuda.synchronize()
net_time += forward_time - pre_process_time
decode_time = time.time()
dec_time += decode_time - forward_time
if self.opt.debug >= 2:
self.debug(debugger, images, dets, output, scale)
dets = self.post_process(dets, meta, scale)
torch.cuda.synchronize()
post_process_time = time.time()
post_time += post_process_time - decode_time
detections.append(dets)
results = self.merge_outputs(detections)
torch.cuda.synchronize()
end_time = time.time()
merge_time += end_time - post_process_time
tot_time += end_time - start_time
if self.opt.debug >= 1:
# print('--->>> base_detector run show_results')
# img_ = self.show_results(debugger, image, results)
debugger.add_img(image, img_id='multi_pose')
#---------------------------------------------------------------- NMS
nms_dets_ = []
for bbox in results[1]:
if bbox[4] > self.opt.vis_thresh:
nms_dets_.append(
(bbox[0], bbox[1], bbox[2], bbox[3], bbox[4]))
if len(nms_dets_) > 0:
keep_ = py_cpu_nms(np.array(nms_dets_), thresh=0.35)
# print('keep_ : ',nms_dets_,keep_)
#----------------------------------------------------------------
faces_boxes = []
person_boxes = []
idx = 0
for bbox in results[1]:
if bbox[4] > self.opt.vis_thresh:
idx += 1
if (idx - 1) not in keep_:
continue
# 绘制目标物体
# print('------------------>>>add_coco_bbox')
debugger.add_coco_bbox(bbox[:4],
0,
bbox[4],
img_id='multi_pose')
face_pts = debugger.add_coco_hp(bbox[5:39],
img_id='multi_pose')
# print('--------------------------------->>>>>>>>>>oou')
if len(face_pts) == 5:
# print('change box')
person_boxes.append([
int(bbox[0]),
int(bbox[1]),
int(bbox[2]),
int(bbox[3]), bbox[4]
])
x_min = min(
[face_pts[i][0] for i in range(len(face_pts))])
y_min = min(
[face_pts[i][1] for i in range(len(face_pts))])
x_max = max(
[face_pts[i][0] for i in range(len(face_pts))])
y_max = max(
[face_pts[i][1] for i in range(len(face_pts))])
edge = abs(x_max - x_min)
#
bbox_x1 = int(max(0, (x_min - edge * 0.05)))
bbox_x2 = int(
min(image.shape[1] - 1, (x_max + edge * 0.05)))
bbox_y1 = int(max(0, (y_min - edge * 0.32)))
bbox_y2 = int(
min(image.shape[0] - 1, (y_max + edge * 0.55)))
# print('ppppp',face_pts,x1)
# if ((bbox_x2-bbox_x1)*(bbox_y2-bbox_y1))>100:
faces_boxes.append(
[bbox_x1, bbox_y1, bbox_x2, bbox_y2, 1.])
# cv2.rectangle(image,(bbox_x1,bbox_y1),(bbox_x2,bbox_y2),(0,255,255),2)
# print('-------->>> show_results debugger')
img_ = debugger.show_all_imgs(pause=self.pause)
return img_, {
'results': results,
'tot': tot_time,
'load': load_time,
'pre': pre_time,
'net': net_time,
'dec': dec_time,
'post': post_time,
'merge': merge_time
}, faces_boxes, person_boxes
