def process(self, images, pre_images=None, pre_hms=None, pre_inds=None, return_time=False): with torch.no_grad(): if self.opt.gpus[0] >= 0: torch.cuda.synchronize() output = self.model(images, pre_images, pre_hms)[-1] output = self._sigmoid_output(output) output.update({'pre_inds': pre_inds}) if self.opt.flip_test: output = self._flip_output(output) if self.opt.gpus[0] >= 0: torch.cuda.synchronize() forward_time = time.time() dets = generic_decode(output, K=self.opt.K, opt=self.opt) if self.opt.gpus[0] >= 0: torch.cuda.synchronize() for k in dets: dets[k] = dets[k].detach().cpu().numpy() if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, pre_images=None, pre_hms=None, return_time=False): with torch.no_grad(): torch.cuda.synchronize() if self.opt.trades: self.inference_prehm.appendleft(pre_hms) while len(self.inference_prehm) < (self.opt.clip_len - 1): self.inference_preim.appendleft(pre_images) self.inference_prehm.appendleft(pre_hms) output, current_feat = self.model(images, pre_images, pre_hms, inference_feats=self.inference_feats, addtional_pre_imgs=self.inference_preim, addtional_pre_hms=self.inference_prehm) output = output[-1] output['tracking'] = output['tracking_offset'] self.inference_feats.appendleft(current_feat) while len(self.inference_feats) < (self.opt.clip_len-1): self.inference_feats.appendleft(current_feat) else: output, current_feat = self.model(images, pre_images, pre_hms) output = output[-1] output = self._sigmoid_output(output) if self.opt.flip_test: output = self._flip_output(output) torch.cuda.synchronize() forward_time = time.time() dets, cur_det = generic_decode(output, K=self.opt.K, opt=self.opt) torch.cuda.synchronize() for k in dets: dets[k] = dets[k].detach().cpu().numpy() if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, pre_images=None, pre_hms=None, pre_inds=None): from model.decode import generic_decode with torch.no_grad(): torch.cuda.synchronize() output = self.model(images, pre_images, pre_hms)[-1] output = self._sigmoid_output(output) output.update({'pre_inds': pre_inds}) if self.opt.flip_test: output = self._flip_output(output) torch.cuda.synchronize() dets = generic_decode(output, K=self.opt.K, opt=self.opt) torch.cuda.synchronize() for k in dets: dets[k] = dets[k].detach().cpu().numpy() return output, dets
def debug(self, batch, output, iter_id, dataset): opt = self.opt if "pre_hm" in batch: output.update({"pre_hm": batch["pre_hm"]}) dets = generic_decode(output, K=opt.K, opt=opt) for k in dets: dets[k] = dets[k].detach().cpu().numpy() dets_gt = batch["meta"]["gt_det"] for i in range(1): debugger = Debugger(opt=opt, dataset=dataset) img = batch["image"][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * dataset.std + dataset.mean) * 255.0), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output["hm"][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch["hm"][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, "pred_hm") debugger.add_blend_img(img, gt, "gt_hm") if "pre_img" in batch: pre_img = batch["pre_img"][i].detach().cpu().numpy().transpose( 1, 2, 0) pre_img = np.clip( ((pre_img * dataset.std + dataset.mean) * 255), 0, 255).astype(np.uint8) debugger.add_img(pre_img, "pre_img_pred") debugger.add_img(pre_img, "pre_img_gt") if "pre_hm" in batch: pre_hm = debugger.gen_colormap( batch["pre_hm"][i].detach().cpu().numpy()) debugger.add_blend_img(pre_img, pre_hm, "pre_hm") debugger.add_img(img, img_id="out_pred") if "ltrb_amodal" in opt.heads: debugger.add_img(img, img_id="out_pred_amodal") debugger.add_img(img, img_id="out_gt_amodal") # Predictions for k in range(len(dets["scores"][i])): if dets["scores"][i, k] > opt.vis_thresh: debugger.add_coco_bbox( dets["bboxes"][i, k] * opt.down_ratio, dets["clses"][i, k], dets["scores"][i, k], img_id="out_pred", ) if "ltrb_amodal" in opt.heads: debugger.add_coco_bbox( dets["bboxes_amodal"][i, k] * opt.down_ratio, dets["clses"][i, k], dets["scores"][i, k], img_id="out_pred_amodal", ) if "hps" in opt.heads and int(dets["clses"][i, k]) == 0: debugger.add_coco_hp(dets["hps"][i, k] * opt.down_ratio, img_id="out_pred") if "tracking" in opt.heads: debugger.add_arrow( dets["cts"][i][k] * opt.down_ratio, dets["tracking"][i][k] * opt.down_ratio, img_id="out_pred", ) debugger.add_arrow( dets["cts"][i][k] * opt.down_ratio, dets["tracking"][i][k] * opt.down_ratio, img_id="pre_img_pred", ) # Ground truth debugger.add_img(img, img_id="out_gt") for k in range(len(dets_gt["scores"][i])): if dets_gt["scores"][i][k] > opt.vis_thresh: debugger.add_coco_bbox( dets_gt["bboxes"][i][k] * opt.down_ratio, dets_gt["clses"][i][k], dets_gt["scores"][i][k], img_id="out_gt", ) if "ltrb_amodal" in opt.heads: debugger.add_coco_bbox( dets_gt["bboxes_amodal"][i, k] * opt.down_ratio, dets_gt["clses"][i, k], dets_gt["scores"][i, k], img_id="out_gt_amodal", ) if "hps" in opt.heads and (int(dets["clses"][i, k]) == 0): debugger.add_coco_hp(dets_gt["hps"][i][k] * opt.down_ratio, img_id="out_gt") if "tracking" in opt.heads: debugger.add_arrow( dets_gt["cts"][i][k] * opt.down_ratio, dets_gt["tracking"][i][k] * opt.down_ratio, img_id="out_gt", ) debugger.add_arrow( dets_gt["cts"][i][k] * opt.down_ratio, dets_gt["tracking"][i][k] * opt.down_ratio, img_id="pre_img_gt", ) if "hm_hp" in opt.heads: pred = debugger.gen_colormap_hp( output["hm_hp"][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp( batch["hm_hp"][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, "pred_hmhp") debugger.add_blend_img(img, gt, "gt_hmhp") if "rot" in opt.heads and "dim" in opt.heads and "dep" in opt.heads: dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt} calib = (batch["meta"]["calib"].detach().numpy() if "calib" in batch["meta"] else None) det_pred = generic_post_process( opt, dets, batch["meta"]["c"].cpu().numpy(), batch["meta"]["s"].cpu().numpy(), output["hm"].shape[2], output["hm"].shape[3], self.opt.num_classes, calib, ) det_gt = generic_post_process( opt, dets_gt, batch["meta"]["c"].cpu().numpy(), batch["meta"]["s"].cpu().numpy(), output["hm"].shape[2], output["hm"].shape[3], self.opt.num_classes, calib, ) debugger.add_3d_detection( batch["meta"]["img_path"][i], batch["meta"]["flipped"][i], det_pred[i], calib[i], vis_thresh=opt.vis_thresh, img_id="add_pred", ) debugger.add_3d_detection( batch["meta"]["img_path"][i], batch["meta"]["flipped"][i], det_gt[i], calib[i], vis_thresh=opt.vis_thresh, img_id="add_gt", ) debugger.add_bird_views( det_pred[i], det_gt[i], vis_thresh=opt.vis_thresh, img_id="bird_pred_gt", ) if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix="{}".format(iter_id)) else: debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id, dataset): opt = self.opt if 'pre_hm' in batch: output.update({'pre_hm': batch['pre_hm']}) dets = generic_decode(output, K=opt.K, opt=opt) for k in dets: dets[k] = dets[k].detach().cpu().numpy() dets_gt = batch['meta']['gt_det'] for i in range(1): debugger = Debugger(opt=opt, dataset=dataset) img = batch['image'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') if 'pre_img' in batch: pre_img = batch['pre_img'][i].detach().cpu().numpy().transpose( 1, 2, 0) pre_img = np.clip( ((pre_img * dataset.std + dataset.mean) * 255), 0, 255).astype(np.uint8) debugger.add_img(pre_img, 'pre_img_pred') debugger.add_img(pre_img, 'pre_img_gt') if 'pre_hm' in batch: pre_hm = debugger.gen_colormap( batch['pre_hm'][i].detach().cpu().numpy()) debugger.add_blend_img(pre_img, pre_hm, 'pre_hm') debugger.add_img(img, img_id='out_pred') if 'ltrb_amodal' in opt.heads: debugger.add_img(img, img_id='out_pred_amodal') debugger.add_img(img, img_id='out_gt_amodal') # Predictions for k in range(len(dets['scores'][i])): if dets['scores'][i, k] > opt.vis_thresh: debugger.add_coco_bbox(dets['bboxes'][i, k] * opt.down_ratio, dets['clses'][i, k], dets['scores'][i, k], img_id='out_pred') if 'ltrb_amodal' in opt.heads: debugger.add_coco_bbox(dets['bboxes_amodal'][i, k] * opt.down_ratio, dets['clses'][i, k], dets['scores'][i, k], img_id='out_pred_amodal') if 'hps' in opt.heads and int(dets['clses'][i, k]) == 0: debugger.add_coco_hp(dets['hps'][i, k] * opt.down_ratio, img_id='out_pred') if 'tracking' in opt.heads: debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio, dets['tracking'][i][k] * opt.down_ratio, img_id='out_pred') debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio, dets['tracking'][i][k] * opt.down_ratio, img_id='pre_img_pred') # Ground truth debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt['scores'][i])): if dets_gt['scores'][i][k] > opt.vis_thresh: debugger.add_coco_bbox(dets_gt['bboxes'][i][k] * opt.down_ratio, dets_gt['clses'][i][k], dets_gt['scores'][i][k], img_id='out_gt') if 'ltrb_amodal' in opt.heads: debugger.add_coco_bbox(dets_gt['bboxes_amodal'][i, k] * opt.down_ratio, dets_gt['clses'][i, k], dets_gt['scores'][i, k], img_id='out_gt_amodal') if 'hps' in opt.heads and \ (int(dets['clses'][i, k]) == 0): debugger.add_coco_hp(dets_gt['hps'][i][k] * opt.down_ratio, img_id='out_gt') if 'tracking' in opt.heads: debugger.add_arrow( dets_gt['cts'][i][k] * opt.down_ratio, dets_gt['tracking'][i][k] * opt.down_ratio, img_id='out_gt') debugger.add_arrow( dets_gt['cts'][i][k] * opt.down_ratio, dets_gt['tracking'][i][k] * opt.down_ratio, img_id='pre_img_gt') if 'hm_hp' in opt.heads: pred = debugger.gen_colormap_hp( output['hm_hp'][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp( batch['hm_hp'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hmhp') debugger.add_blend_img(img, gt, 'gt_hmhp') if 'rot' in opt.heads and 'dim' in opt.heads and 'dep' in opt.heads: dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt} calib = batch['meta']['calib'].detach().numpy() \ if 'calib' in batch['meta'] else None det_pred = generic_post_process( opt, dets, batch['meta']['c'].cpu().numpy(), batch['meta']['s'].cpu().numpy(), output['hm'].shape[2], output['hm'].shape[3], self.opt.num_classes, calib) det_gt = generic_post_process(opt, dets_gt, batch['meta']['c'].cpu().numpy(), batch['meta']['s'].cpu().numpy(), output['hm'].shape[2], output['hm'].shape[3], self.opt.num_classes, calib) debugger.add_3d_detection(batch['meta']['img_path'][i], batch['meta']['flipped'][i], det_pred[i], calib[i], vis_thresh=opt.vis_thresh, img_id='add_pred') debugger.add_3d_detection(batch['meta']['img_path'][i], batch['meta']['flipped'][i], det_gt[i], calib[i], vis_thresh=opt.vis_thresh, img_id='add_gt') debugger.add_bird_views(det_pred[i], det_gt[i], vis_thresh=opt.vis_thresh, img_id='bird_pred_gt') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)