def display_map(self, batch, tracks, idx): opt = self.opt for i in range(1): debugger = Debugger(opt, dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) debugger.add_img(img, img_id='track') for i in range(len(tracks)): dets = tracks[i].pred bbox = dets[:4] * self.opt.down_ratio w, h = bbox[2], bbox[3] bbox = np.array([ bbox[0] - w / 2, bbox[1] - h / 2, bbox[0] + w / 2, bbox[1] + h / 2 ]) debugger.add_coco_bbox(bbox, int(dets[-1]), tracks[i].track_id, img_id='track', tracking=True) debugger.save_all_imgs(opt.debug_dir, prefix=f'{idx}')
def debug(self, batch, output, iter_id): opt = self.opt reg = output['reg'] if opt.reg_offset else None dets = ctdet_decode(output['hm'], output['wh'], reg=reg, cat_spec_wh=opt.cat_spec_wh, opt=opt) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= opt.down_ratio dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) dets_gt[:, :, :4] *= opt.down_ratio for i in range(1): debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') if opt.edge_hm: edge_hm = output['edge_hm'][i].detach().cpu().numpy() edge_hm = edge_hm.reshape(4 * opt.num_edge_hm, -1, edge_hm.shape[1], edge_hm.shape[2]) edge_hm = edge_hm.sum(axis=0) edge_hm = debugger.gen_colormap(edge_hm) debugger.add_blend_img(img, edge_hm, 'edge_hm') gt_edge_hm = batch['edge_hm'][i].detach().cpu().numpy() gt_edge_hm = gt_edge_hm.reshape(4 * opt.num_edge_hm, -1, gt_edge_hm.shape[1], gt_edge_hm.shape[2]) gt_edge_hm = gt_edge_hm.sum(axis=0) gt_edge_hm = debugger.gen_colormap(gt_edge_hm) debugger.add_blend_img(img, gt_edge_hm, 'gt_edge_hm') for k in range(len(dets[i])): if dets[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id): opt = self.opt reg = output['reg'] if opt.reg_offset else None dets = ctdet_decode(output['hm'], output['wh'], reg=reg, cat_spec_wh=opt.cat_spec_wh, K=opt.K) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= opt.down_ratio dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) dets_gt[:, :, :4] *= opt.down_ratio if opt.task == 'ctdet_semseg': seg_gt = batch['seg'][0][0].cpu().numpy() seg_pred = output['seg'].max(1)[1].squeeze_(1).squeeze_( 0).cpu().numpy() for i in range(1): debugger = Debugger(opt, dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) debugger.add_img(img, img_id='out_pred') for k in range(len(dets[i])): if dets[i, k, 4] > opt.vis_thresh: debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.vis_thresh: debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') if opt.save_video: # only save the predicted and gt images return debugger.imgs['out_pred'], debugger.imgs['out_gt'] pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') if opt.task == 'ctdet_semseg': debugger.visualize_masks(seg_gt, img_id='out_mask_gt') debugger.visualize_masks(seg_pred, img_id='out_mask_pred') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix=iter_id)
def debug(self, batch, output, iter_id): opt = self.opt ###是否进行坐标offset reg reg = output['reg'] if opt.reg_offset else None ###将网络输出的hms经过decode得到detections: [bboxes, scores, clses] dets = ctdet_decode( output['hm'], output['wh'], reg=reg, cat_spec_wh=opt.cat_spec_wh, K=opt.K) ####创建一个没有梯度的变量dets,shape为(1,batch*k,6) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) ####对预测坐标进行变换---->下采样, down_ratio默认值为4 dets[:, :, :4] *= opt.down_ratio ####把dets_gt的shape变为(1,batch*k, 6) ####dets_gt为gt_bbox的位置信息,shape为(1,batch*k,6) dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) ####对gt坐标进行变换---->下采样, down_ratio默认值为4 dets_gt[:, :, :4] *= opt.down_ratio for i in range(1): debugger = Debugger( dataset=opt.dataset, ipynb=(opt.debug==3), theme=opt.debugger_theme) ###将输入图片转化为cpu上的没有梯度的张量img img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) ###对输入图像进行标准化处理:乘上标准差再加上均值 img = np.clip((( img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) ####gen_colormap又是什么玩意??? ####output----->pred, batch------>gt pred = debugger.gen_colormap(output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) ####add_blend_img是用来干嘛??? debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') ###此时len(dets[i]==dets[0])==batch*k, for k in range(len(dets[i])): ###即某个score>thresh if dets[i, k, 4] > opt.center_thresh: ####在图像上画出检测框,坐标,score和cls debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_img(img, img_id='out_gt') ###len(dets_gt[i])为batch*k for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.center_thresh: ####画出gt_bbox debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id): opt = self.opt reg = output['reg'] if opt.reg_offset else None hm_hp = output['hm_hp'] if opt.hm_hp else None hp_offset = output['hp_offset'] if opt.reg_hp_offset else None dets = multi_pose_decode( output['hm'], output['wh'], output['hps'], reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=opt.K) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= opt.input_res / opt.output_res dets[:, :, 5:39] *= opt.input_res / opt.output_res dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) dets_gt[:, :, :4] *= opt.input_res / opt.output_res dets_gt[:, :, 5:39] *= opt.input_res / opt.output_res for i in range(1): debugger = Debugger( dataset=opt.dataset, ipynb=(opt.debug==3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip((( img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap(output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') for k in range(len(dets[i])): if dets[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_coco_hp(dets[i, k, 5:39], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') debugger.add_coco_hp(dets_gt[i, k, 5:39], img_id='out_gt') if opt.hm_hp: pred = debugger.gen_colormap_hp(output['hm_hp'][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp(batch['hm_hp'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hmhp') debugger.add_blend_img(img, gt, 'gt_hmhp') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id): cfg = self.cfg reg = output[3] if cfg.LOSS.REG_OFFSET else None hm_hp = output[4] if cfg.LOSS.HM_HP else None hp_offset = output[5] if cfg.LOSS.REG_HP_OFFSET else None dets = multi_pose_decode( output[0], output[1], output[2], reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=cfg.TEST.TOPK) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES dets[:, :, 5:39] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) dets_gt[:, :, :4] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES dets_gt[:, :, 5:39] *= cfg.MODEL.INPUT_RES / cfg.MODEL.OUTPUT_RES for i in range(1): debugger = Debugger( dataset=cfg.SAMPLE_METHOD, ipynb=(cfg.DEBUG==3), theme=cfg.DEBUG_THEME) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip((( img * np.array(cfg.DATASET.STD).reshape(1,1,3).astype(np.float32) + cfg.DATASET.MEAN) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap(output[0][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') for k in range(len(dets[i])): if dets[i, k, 4] > cfg.MODEL.CENTER_THRESH: debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_coco_hp(dets[i, k, 5:39], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > cfg.MODEL.CENTER_THRESH: debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') debugger.add_coco_hp(dets_gt[i, k, 5:39], img_id='out_gt') if cfg.LOSS.HM_HP: pred = debugger.gen_colormap_hp(output[4][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp(batch['hm_hp'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hmhp') debugger.add_blend_img(img, gt, 'gt_hmhp') if cfg.DEBUG == 4: debugger.save_all_imgs(cfg.LOG_DIR, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id): opt = self.opt detections = self.decode(output['hm_t'], output['hm_l'], output['hm_b'], output['hm_r'], output['hm_c']).detach().cpu().numpy() detections[:, :, :4] *= opt.input_res / opt.output_res for i in range(1): dataset = opt.dataset if opt.dataset == 'yolo': dataset = opt.names debugger = Debugger(dataset=dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) pred_hm = np.zeros((opt.input_res, opt.input_res, 3), dtype=np.uint8) gt_hm = np.zeros((opt.input_res, opt.input_res, 3), dtype=np.uint8) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = ((img * self.opt.std + self.opt.mean) * 255.).astype( np.uint8) for p in self.parts: tag = 'hm_{}'.format(p) pred = debugger.gen_colormap( output[tag][i].detach().cpu().numpy()) gt = debugger.gen_colormap( batch[tag][i].detach().cpu().numpy()) if p != 'c': pred_hm = np.maximum(pred_hm, pred) gt_hm = np.maximum(gt_hm, gt) if p == 'c' or opt.debug > 2: debugger.add_blend_img(img, pred, 'pred_{}'.format(p)) debugger.add_blend_img(img, gt, 'gt_{}'.format(p)) debugger.add_blend_img(img, pred_hm, 'pred') debugger.add_blend_img(img, gt_hm, 'gt') debugger.add_img(img, img_id='out') for k in range(len(detections[i])): if detections[i, k, 4] > 0.1: debugger.add_coco_bbox(detections[i, k, :4], detections[i, k, -1], detections[i, k, 4], img_id='out') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def show_results(self, save_dir): with open(os.path.join(save_dir, "results.json")) as f: data = json.load(f) debugger = Debugger(dataset=self.opt.dataset, ipynb=(self.opt.debug == 3), theme=self.opt.debugger_theme, class_names=self.opt.class_names) for i, img_details in enumerate(data): if (data[i]['score'] > self.opt.vis_thresh): img_path = os.path.join("../data/coco/test2019", str(img_details["image_id"]) + ".jpg") # TODO image = cv2.imread(img_path) debugger.add_img(image, img_id='ctdet') bbox = data[i]['bbox'] debugger.add_coco_bbox(bbox, data[i]['category_id'] - 1, data[i]['score'], img_id='ctdet') debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id): opt = self.opt reg = output['reg'] if opt.reg_offset else None dets = ctdet_decode(output['hm'], output['wh'], reg=reg, cat_spec_wh=opt.cat_spec_wh, K=opt.K) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= opt.down_ratio dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) dets_gt[:, :, :4] *= opt.down_ratio for i in range(1): debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') for k in range(len(dets[i])): if dets[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt')
def debug(self, batch, output, iter_id, dataset): opt = self.opt if 'pre_hm' in batch: output.update({'pre_hm': batch['pre_hm']}) dets = fusion_decode(output, K=opt.K, opt=opt) for k in dets: dets[k] = dets[k].detach().cpu().numpy() dets_gt = batch['meta']['gt_det'] for i in range(1): debugger = Debugger(opt=opt, dataset=dataset) img = batch['image'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm', trans=self.opt.hm_transparency) debugger.add_blend_img(img, gt, 'gt_hm', trans=self.opt.hm_transparency) debugger.add_img(img, img_id='img') # show point clouds if opt.pointcloud: pc_2d = batch['pc_2d'][i].detach().cpu().numpy() pc_3d = None pc_N = batch['pc_N'][i].detach().cpu().numpy() debugger.add_img(img, img_id='pc') debugger.add_pointcloud(pc_2d, pc_N, img_id='pc') if 'pc_hm' in opt.pc_feat_lvl: channel = opt.pc_feat_channels['pc_hm'] pc_hm = debugger.gen_colormap( batch['pc_hm'][i][channel].unsqueeze( 0).detach().cpu().numpy()) debugger.add_blend_img(img, pc_hm, 'pc_hm', trans=self.opt.hm_transparency) if 'pc_dep' in opt.pc_feat_lvl: channel = opt.pc_feat_channels['pc_dep'] pc_hm = batch['pc_hm'][i][channel].unsqueeze( 0).detach().cpu().numpy() pc_dep = debugger.add_overlay_img(img, pc_hm, 'pc_dep') if 'pre_img' in batch: pre_img = batch['pre_img'][i].detach().cpu().numpy().transpose( 1, 2, 0) pre_img = np.clip( ((pre_img * dataset.std + dataset.mean) * 255), 0, 255).astype(np.uint8) debugger.add_img(pre_img, 'pre_img_pred') debugger.add_img(pre_img, 'pre_img_gt') if 'pre_hm' in batch: pre_hm = debugger.gen_colormap( batch['pre_hm'][i].detach().cpu().numpy()) debugger.add_blend_img(pre_img, pre_hm, 'pre_hm', trans=self.opt.hm_transparency) debugger.add_img(img, img_id='out_pred') if 'ltrb_amodal' in opt.heads: debugger.add_img(img, img_id='out_pred_amodal') debugger.add_img(img, img_id='out_gt_amodal') # Predictions for k in range(len(dets['scores'][i])): if dets['scores'][i, k] > opt.vis_thresh: debugger.add_coco_bbox(dets['bboxes'][i, k] * opt.down_ratio, dets['clses'][i, k], dets['scores'][i, k], img_id='out_pred') if 'ltrb_amodal' in opt.heads: debugger.add_coco_bbox(dets['bboxes_amodal'][i, k] * opt.down_ratio, dets['clses'][i, k], dets['scores'][i, k], img_id='out_pred_amodal') if 'hps' in opt.heads and int(dets['clses'][i, k]) == 0: debugger.add_coco_hp(dets['hps'][i, k] * opt.down_ratio, img_id='out_pred') if 'tracking' in opt.heads: debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio, dets['tracking'][i][k] * opt.down_ratio, img_id='out_pred') debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio, dets['tracking'][i][k] * opt.down_ratio, img_id='pre_img_pred') # Ground truth debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt['scores'][i])): if dets_gt['scores'][i][k] > opt.vis_thresh: if 'dep' in dets_gt.keys(): dist = dets_gt['dep'][i][k] if len(dist) > 1: dist = dist[0] else: dist = -1 debugger.add_coco_bbox(dets_gt['bboxes'][i][k] * opt.down_ratio, dets_gt['clses'][i][k], dets_gt['scores'][i][k], img_id='out_gt', dist=dist) if 'ltrb_amodal' in opt.heads: debugger.add_coco_bbox(dets_gt['bboxes_amodal'][i, k] * opt.down_ratio, dets_gt['clses'][i, k], dets_gt['scores'][i, k], img_id='out_gt_amodal') if 'hps' in opt.heads and \ (int(dets['clses'][i, k]) == 0): debugger.add_coco_hp(dets_gt['hps'][i][k] * opt.down_ratio, img_id='out_gt') if 'tracking' in opt.heads: debugger.add_arrow( dets_gt['cts'][i][k] * opt.down_ratio, dets_gt['tracking'][i][k] * opt.down_ratio, img_id='out_gt') debugger.add_arrow( dets_gt['cts'][i][k] * opt.down_ratio, dets_gt['tracking'][i][k] * opt.down_ratio, img_id='pre_img_gt') if 'hm_hp' in opt.heads: pred = debugger.gen_colormap_hp( output['hm_hp'][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp( batch['hm_hp'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hmhp', trans=self.opt.hm_transparency) debugger.add_blend_img(img, gt, 'gt_hmhp', trans=self.opt.hm_transparency) if 'rot' in opt.heads and 'dim' in opt.heads and 'dep' in opt.heads: dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt} calib = batch['meta']['calib'].detach().numpy() \ if 'calib' in batch['meta'] else None det_pred = generic_post_process( opt, dets, batch['meta']['c'].cpu().numpy(), batch['meta']['s'].cpu().numpy(), output['hm'].shape[2], output['hm'].shape[3], self.opt.num_classes, calib) det_gt = generic_post_process(opt, dets_gt, batch['meta']['c'].cpu().numpy(), batch['meta']['s'].cpu().numpy(), output['hm'].shape[2], output['hm'].shape[3], self.opt.num_classes, calib, is_gt=True) debugger.add_3d_detection(batch['meta']['img_path'][i], batch['meta']['flipped'][i], det_pred[i], calib[i], vis_thresh=opt.vis_thresh, img_id='add_pred') debugger.add_3d_detection(batch['meta']['img_path'][i], batch['meta']['flipped'][i], det_gt[i], calib[i], vis_thresh=opt.vis_thresh, img_id='add_gt') pc_3d = None if opt.pointcloud: pc_3d = batch['pc_3d'].cpu().numpy() debugger.add_bird_views(det_pred[i], det_gt[i], vis_thresh=opt.vis_thresh, img_id='bird_pred_gt', pc_3d=pc_3d, show_velocity=opt.show_velocity) debugger.add_bird_views([], det_gt[i], vis_thresh=opt.vis_thresh, img_id='bird_gt', pc_3d=pc_3d, show_velocity=opt.show_velocity) if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def detect(opt): os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str split = 'val' if not opt.trainval else 'test' dataset = YOLO(opt.data_dir, opt.flip, opt.vflip, opt.rotate, opt.scale, opt.shear, opt, split) opt = opts().update_dataset_info_and_set_heads(opt, dataset) print(opt) # log = Logger(opt) Detector = detector_factory[opt.task] detector = Detector(opt) debugger = Debugger(dataset=opt.names) dir_path = os.path.join(opt.save_dir, 'detect') if not os.path.exists(dir_path): os.mkdir(dir_path) images = [] if os.path.isfile(opt.image): if os.path.splitext(opt.image)[1] == '.txt': name = os.path.splitext(os.path.basename(opt.image))[0] dir_path = os.path.join(dir_path, name) if not os.path.exists(dir_path): os.mkdir(dir_path) with open(opt.image, 'r') as f: images.extend([l.rstrip().replace('.txt', '.jpg') for l in f.readlines()]) elif os.path.splitext(opt.image)[1] in ['.jpg', '.png', '.bmp']: images.append(opt.image) else: raise Exception('NOT SUPPORT FILE TYPE!!!') else: for file in os.listdir(opt.image): if os.path.splitext(file)[1] in ['.jpg', '.png', '.bmp']: images.append(os.path.join(opt.image, file)) num_iters = len(images) bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind in range(num_iters): img_id = images[ind] ret = detector.run(img_id) Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + '|{} {tm.val:.3f}s ({tm.avg:.3f}s) '.format( t, tm=avg_time_stats[t]) bar.next() img_name = os.path.splitext(os.path.basename(img_id))[0] img = cv2.imread(img_id) h, w = img.shape[:2] pred = debugger.gen_colormap(ret['output']['hm'][0].detach().cpu().numpy()) debugger.add_blend_img(img, pred, img_name+'pred_hm') debugger.add_img(img, img_id=img_name) gt = np.loadtxt(img_id.replace('.jpg', '.txt')).reshape(-1, 5) if gt.size: x1 = w * (gt[:, 1] - gt[:, 3] / 2) y1 = h * (gt[:, 2] - gt[:, 4] / 2) x2 = w * (gt[:, 1] + gt[:, 3] / 2) y2 = h * (gt[:, 2] + gt[:, 4] / 2) gt[:, 1] = x1 gt[:, 2] = y1 gt[:, 3] = x2 gt[:, 4] = y2 for g in gt: debugger.add_gt_bbox(g, img_id=img_name) path = os.path.join(dir_path, os.path.basename(img_id).replace('.jpg', '.txt')) dets = np.zeros((0, 6), dtype=np.float32) for cls, det in ret['results'].items(): cls_id = np.ones((len(det), 1), dtype=np.float32) * (cls - 1) dets = np.append(dets, np.hstack((det, cls_id)), 0) for d in det: if d[-1] >= opt.vis_thresh: debugger.add_coco_bbox(d[:4], cls-1, d[-1], img_id=img_name) np.savetxt(path, dets) bar.finish() debugger.save_all_imgs(path=dir_path)
def debug(self, batch, output, iter_id, dataset): opt = self.opt if 'pre_hm' in batch: output.update({'pre_hm': batch['pre_hm']}) dets = generic_decode(output, K=opt.K, opt=opt) for k in dets: dets[k] = dets[k].detach().cpu().numpy() dets_gt = batch['meta']['gt_det'] for i in range(1): debugger = Debugger(opt=opt, dataset=dataset) img = batch['image'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') if 'pre_img' in batch: pre_img = batch['pre_img'][i].detach().cpu().numpy().transpose( 1, 2, 0) pre_img = np.clip( ((pre_img * dataset.std + dataset.mean) * 255), 0, 255).astype(np.uint8) debugger.add_img(pre_img, 'pre_img_pred') debugger.add_img(pre_img, 'pre_img_gt') if 'pre_hm' in batch: pre_hm = debugger.gen_colormap( batch['pre_hm'][i].detach().cpu().numpy()) debugger.add_blend_img(pre_img, pre_hm, 'pre_hm') debugger.add_img(img, img_id='out_pred') if 'ltrb_amodal' in opt.heads: debugger.add_img(img, img_id='out_pred_amodal') debugger.add_img(img, img_id='out_gt_amodal') # Predictions for k in range(len(dets['scores'][i])): if dets['scores'][i, k] > opt.vis_thresh: debugger.add_coco_bbox(dets['bboxes'][i, k] * opt.down_ratio, dets['clses'][i, k], dets['scores'][i, k], img_id='out_pred') if 'ltrb_amodal' in opt.heads: debugger.add_coco_bbox(dets['bboxes_amodal'][i, k] * opt.down_ratio, dets['clses'][i, k], dets['scores'][i, k], img_id='out_pred_amodal') if 'hps' in opt.heads and int(dets['clses'][i, k]) == 0: debugger.add_coco_hp(dets['hps'][i, k] * opt.down_ratio, img_id='out_pred') if 'tracking' in opt.heads: debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio, dets['tracking'][i][k] * opt.down_ratio, img_id='out_pred') debugger.add_arrow(dets['cts'][i][k] * opt.down_ratio, dets['tracking'][i][k] * opt.down_ratio, img_id='pre_img_pred') # Ground truth debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt['scores'][i])): if dets_gt['scores'][i][k] > opt.vis_thresh: debugger.add_coco_bbox(dets_gt['bboxes'][i][k] * opt.down_ratio, dets_gt['clses'][i][k], dets_gt['scores'][i][k], img_id='out_gt') if 'ltrb_amodal' in opt.heads: debugger.add_coco_bbox(dets_gt['bboxes_amodal'][i, k] * opt.down_ratio, dets_gt['clses'][i, k], dets_gt['scores'][i, k], img_id='out_gt_amodal') if 'hps' in opt.heads and \ (int(dets['clses'][i, k]) == 0): debugger.add_coco_hp(dets_gt['hps'][i][k] * opt.down_ratio, img_id='out_gt') if 'tracking' in opt.heads: debugger.add_arrow( dets_gt['cts'][i][k] * opt.down_ratio, dets_gt['tracking'][i][k] * opt.down_ratio, img_id='out_gt') debugger.add_arrow( dets_gt['cts'][i][k] * opt.down_ratio, dets_gt['tracking'][i][k] * opt.down_ratio, img_id='pre_img_gt') if 'hm_hp' in opt.heads: pred = debugger.gen_colormap_hp( output['hm_hp'][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp( batch['hm_hp'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hmhp') debugger.add_blend_img(img, gt, 'gt_hmhp') if 'rot' in opt.heads and 'dim' in opt.heads and 'dep' in opt.heads: dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt} calib = batch['meta']['calib'].detach().numpy() \ if 'calib' in batch['meta'] else None det_pred = generic_post_process( opt, dets, batch['meta']['c'].cpu().numpy(), batch['meta']['s'].cpu().numpy(), output['hm'].shape[2], output['hm'].shape[3], self.opt.num_classes, calib) det_gt = generic_post_process(opt, dets_gt, batch['meta']['c'].cpu().numpy(), batch['meta']['s'].cpu().numpy(), output['hm'].shape[2], output['hm'].shape[3], self.opt.num_classes, calib) debugger.add_3d_detection(batch['meta']['img_path'][i], batch['meta']['flipped'][i], det_pred[i], calib[i], vis_thresh=opt.vis_thresh, img_id='add_pred') debugger.add_3d_detection(batch['meta']['img_path'][i], batch['meta']['flipped'][i], det_gt[i], calib[i], vis_thresh=opt.vis_thresh, img_id='add_gt') debugger.add_bird_views(det_pred[i], det_gt[i], vis_thresh=opt.vis_thresh, img_id='bird_pred_gt') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def test_loader(cfg): debugger = Debugger((cfg.DEBUG == 3), theme=cfg.DEBUG_THEME, num_classes=cfg.MODEL.NUM_CLASSES, dataset=cfg.SAMPLE_METHOD, down_ratio=cfg.MODEL.DOWN_RATIO) Dataset = get_dataset(cfg.SAMPLE_METHOD, cfg.TASK) val_dataset = Dataset(cfg, 'val') val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=True) for i, batch_data in enumerate(val_loader): input_image = batch_data['input'] heat = batch_data['hm'] reg = batch_data['reg'] reg_mask = batch_data['reg_mask'] ind = batch_data['ind'] wh = batch_data['wh'] kps = batch_data['hps'] hps_mask = batch_data['hps_mask'] seg_feat = batch_data['seg'] hm_hp = batch_data['hm_hp'] hp_offset = batch_data['hp_offset'] hp_ind = batch_data['hp_ind'] hp_mask = batch_data['hp_mask'] meta = batch_data['meta'] for k, v in batch_data.items(): if type(v) == type(dict()): for k1, v1 in v.items(): print(k1) print(v1) else: print(k) print(v.shape) print(input_image.shape) print(hm_hp.shape) #handle image input_image = input_image[0].numpy().transpose(1, 2, 0) input_image = (input_image * STD) + MEAN input_image = input_image * 255 input_image = input_image.astype(np.uint8) heat = heat.sigmoid_() hm_hp = hm_hp.sigmoid_() num_joints = 17 K = cfg.TEST.TOPK # perform nms on heatmaps batch, cat, height, width = heat.size() heat = _nms(heat) scores, inds, clses, ys, xs = _topk(heat, K=K) kps = kps.view(batch, K, num_joints * 2) kps[..., ::2] += xs.view(batch, K, 1).expand(batch, K, num_joints) kps[..., 1::2] += ys.view(batch, K, 1).expand(batch, K, num_joints) xs = xs.view(batch, K, 1) + reg[:, :, 0:1] ys = ys.view(batch, K, 1) + reg[:, :, 1:2] wh = wh.view(batch, K, 2) #weight = _transpose_and_gather_feat(seg, inds) ## you can write (if weight.size(1)!=seg_feat.size(1): 3x3conv else 1x1conv ) here to select seg conv. ## for 3x3 #weight = weight.view([weight.size(1), -1, 3, 3]) pred_seg = seg_feat clses = clses.view(batch, K, 1).float() scores = scores.view(batch, K, 1) bboxes = torch.cat([ xs - wh[..., 0:1] / 2, ys - wh[..., 1:2] / 2, xs + wh[..., 0:1] / 2, ys + wh[..., 1:2] / 2 ], dim=2) if hm_hp is not None: hm_hp = _nms(hm_hp) thresh = 0.1 kps = kps.view(batch, K, num_joints, 2).permute(0, 2, 1, 3).contiguous() # b x J x K x 2 reg_kps = kps.unsqueeze(3).expand(batch, num_joints, K, K, 2) hm_score, hm_inds, hm_ys, hm_xs = _topk_channel(hm_hp, K=K) # b x J x K hp_offset = hp_offset.view(batch, num_joints, K, 2) hm_xs = hm_xs + hp_offset[:, :, :, 0] hm_ys = hm_ys + hp_offset[:, :, :, 1] mask = (hm_score > thresh).float() hm_score = (1 - mask) * -1 + mask * hm_score hm_ys = (1 - mask) * (-10000) + mask * hm_ys hm_xs = (1 - mask) * (-10000) + mask * hm_xs hm_kps = torch.stack([hm_xs, hm_ys], dim=-1).unsqueeze(2).expand( batch, num_joints, K, K, 2) dist = (((reg_kps - hm_kps)**2).sum(dim=4)**0.5) min_dist, min_ind = dist.min(dim=3) # b x J x K hm_score = hm_score.gather(2, min_ind).unsqueeze(-1) # b x J x K x 1 min_dist = min_dist.unsqueeze(-1) min_ind = min_ind.view(batch, num_joints, K, 1, 1).expand(batch, num_joints, K, 1, 2) hm_kps = hm_kps.gather(3, min_ind) hm_kps = hm_kps.view(batch, num_joints, K, 2) l = bboxes[:, :, 0].view(batch, 1, K, 1).expand(batch, num_joints, K, 1) t = bboxes[:, :, 1].view(batch, 1, K, 1).expand(batch, num_joints, K, 1) r = bboxes[:, :, 2].view(batch, 1, K, 1).expand(batch, num_joints, K, 1) b = bboxes[:, :, 3].view(batch, 1, K, 1).expand(batch, num_joints, K, 1) mask = (hm_kps[..., 0:1] < l) + (hm_kps[..., 0:1] > r) + \ (hm_kps[..., 1:2] < t) + (hm_kps[..., 1:2] > b) + \ (hm_score < thresh) + (min_dist > (torch.max(b - t, r - l) * 0.3)) mask = (mask > 0).float().expand(batch, num_joints, K, 2) kps = (1 - mask) * hm_kps + mask * kps kps = kps.permute(0, 2, 1, 3).contiguous().view(batch, K, num_joints * 2) dets = torch.cat([ bboxes, scores, kps, torch.transpose(hm_score.squeeze(dim=3), 1, 2) ], dim=2) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets, inds = whole_body_post_process(dets.copy(), [meta['c'].numpy()], [meta['s'].numpy()], 128, 128, 1) for j in range(1, cfg.MODEL.NUM_CLASSES + 1): dets[0][j] = np.array(dets[0][j], dtype=np.float32).reshape(-1, 56) dets[0][j][:, :4] /= 1. dets[0][j][:, 5:39] /= 1. print(pred_seg.shape) seg = pred_seg[0] trans = get_affine_transform(meta['c'], meta['s'], 0, (meta['out_width'], meta['out_height']), inv=1) debugger.add_img(image, img_id='multi_pose') for j in range(1, self.num_classes + 1): for b_id, detection in enumerate(results[j]): bbox = detection[:4] bbox_prob = detection[4] keypoints = detection[5:39] keypoints_prob = detection[39:] if bbox_prob > self.cfg.TEST.VIS_THRESH: debugger.add_coco_bbox(bbox, 0, bbox_prob, img_id='multi_pose') segment = seg[b_id].detach().cpu().numpy() segment = cv2.warpAffine(segment, trans, (image.shape[1], image.shape[0]), flags=cv2.INTER_CUBIC) w, h = bbox[2:4] - bbox[:2] ct = np.array([(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2], dtype=np.float32) segment_mask = np.zeros_like(segment) pad_rate = 0.3 x, y = np.clip([ct[0] - (1 + pad_rate) * w / 2, ct[0] + (1 + pad_rate) * w / 2], 0, segment.shape[1] - 1).astype(np.int), \ np.clip([ct[1] - (1 + pad_rate) * h / 2, ct[1] + (1 + pad_rate) * h / 2], 0, segment.shape[0] - 1).astype(np.int) segment_mask[y[0]:y[1], x[0]:x[1]] = 1 segment = segment_mask * segment debugger.add_coco_seg(segment, img_id='multi_pose') debugger.add_coco_hp(keypoints, keypoints_prob, img_id='multi_pose') debugger.show_all_imgs(pause=self.pause) save_path = os.path.join(SAVE_DIR, '{}.png'.format(i)) cv2.imwrite(save_path, input_image)
def demo(opt): class_map = {1: 1, 2: 2} # color for boundingbox os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str # opt.debug = max(opt.debug, 1) Detector = detector_factory[opt.task] detector = Detector(opt) assert os.path.isdir(opt.demo), 'Need path to videos directory.' video_paths = [ os.path.join(opt.demo, video_name) for video_name in os.listdir(opt.demo) if video_name.split('.')[-1] == 'mp4' ] # video_paths = [ # os.path.join(opt.demo, 'cam_2.mp4') # ] debugger = Debugger(dataset=opt.dataset, theme=opt.debugger_theme) for video_path in sorted(video_paths): print('video_name = ', video_path) bboxes = [] video = cv2.VideoCapture(video_path) width, height = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)), int( video.get(cv2.CAP_PROP_FRAME_HEIGHT)) # pointer pts = [] arr_name = os.path.basename(video_path).split('.')[0].split('_') cam_name = arr_name[0] + '_' + arr_name[1] print('cam_name = ', cam_name) with open('../ROIs/{}.txt'.format(cam_name)) as f: for line in f: pts.append([int(x) for x in line.split(',')]) pts = np.array(pts) # make mask mask = np.zeros((height, width), np.uint8) cv2.drawContours(mask, [pts], -1, (255, 255, 255), -1, cv2.LINE_AA) bbox_video = cv2.VideoWriter( filename='/home/leducthinh0409/centernet_visualize_{}/'.format( opt.arch) + os.path.basename(video_path), fourcc=cv2.VideoWriter_fourcc(*'mp4v'), fps=float(30), frameSize=(width, height), isColor=True) num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) for i in tqdm(range(num_frames)): _, img_pre = video.read() ## do bit-op dst = cv2.bitwise_and(img_pre, img_pre, mask=mask) ## add the white background bg = np.ones_like(img_pre, np.uint8) * 255 cv2.bitwise_not(bg, bg, mask=mask) img = bg + dst ret = detector.run(img) bboxes.append(ret['results']) debugger.add_img(img, img_id='default') for class_id in class_map.keys(): for bbox in ret['results'][class_id]: if bbox[4] > opt.vis_thresh: debugger.add_coco_bbox(bbox[:4], class_map[class_id], bbox[4], img_id='default') bbox_img = debugger.imgs['default'] bbox_video.write(bbox_img) with open( '/home/leducthinh0409/bboxes_{}/'.format(opt.arch) + os.path.basename(video_path) + '.pkl', 'wb') as f: pickle.dump(bboxes, f)
def train(self, cfg): # 设置gpu环境,考虑单卡多卡情况 gpus_str = '' if isinstance(cfg.gpus, (list, tuple)): cfg.gpus = [int(i) for i in cfg.gpus] for s in cfg.gpus: gpus_str += str(s) + ',' gpus_str = gpus_str[:-1] else: gpus_str = str(int(cfg.gpus)) cfg.gpus = [int(cfg.gpus)] os.environ['CUDA_VISIBLE_DEVICES'] = gpus_str cfg.gpus = [i for i in range(len(cfg.gpus)) ] if cfg.gpus[0] >= 0 else [-1] # 设置log model_dir = os.path.join(cfg.save_dir, cfg.id) debug_dir = os.path.join(model_dir, 'debug') if not os.path.exists(model_dir): os.makedirs(model_dir) if not os.path.exists(debug_dir): os.makedirs(debug_dir) logger = setup_logger(cfg.id, os.path.join(model_dir, 'log')) if USE_TENSORBOARD: writer = tensorboardX.SummaryWriter( log_dir=os.path.join(model_dir, 'log')) logger.info(cfg) gpus = cfg.gpus device = torch.device('cpu' if gpus[0] < 0 else 'cuda') lr = cfg.lr lr_step = cfg.lr_step num_epochs = cfg.num_epochs val_step = cfg.val_step sample_size = cfg.sample_size # 设置数据集 dataset = YOLO(cfg.data_dir, cfg.hflip, cfg.vflip, cfg.rotation, cfg.scale, cfg.shear, opt=cfg, split='train') names = dataset.class_name std = dataset.std mean = dataset.mean # 用数据集类别数设置预测网络 cfg.setup_head(dataset) trainloader = DataLoader(dataset, batch_size=cfg.batch_size, shuffle=True, num_workers=cfg.num_workers, pin_memory=True, drop_last=True) # val_dataset = YOLO(cfg.data_dir, cfg.hflip, cfg.vflip, cfg.rotation, cfg.scale, cfg.shear, opt=cfg, split='val') # valloader = DataLoader(val_dataset, batch_size=1, shuffle=True, num_workers=1, pin_memory=True) valid_file = cfg.val_dir if not cfg.val_dir == '' else os.path.join( cfg.data_dir, 'valid.txt') with open(valid_file, 'r') as f: val_list = [l.rstrip() for l in f.readlines()] net = create_model(cfg.arch, cfg.heads, cfg.head_conv, cfg.down_ratio, cfg.filters) optimizer = optim.Adam(net.parameters(), lr=lr) start_epoch = 0 if cfg.resume: pretrain = os.path.join(model_dir, 'model_last.pth') if os.path.exists(pretrain): print('resume model from %s' % pretrain) try: net, optimizer, start_epoch = load_model( net, pretrain, optimizer, True, lr, lr_step) except: print('\t... loading model error: ckpt may not compatible') model = ModleWithLoss(net, CtdetLoss(cfg)) if len(gpus) > 1: model = nn.DataParallel(model, device_ids=gpus).to(device) else: model = model.to(device) step = 0 best = 1e10 log_loss_stats = ['loss', 'hm_loss', 'wh_loss'] if cfg.reg_offset: log_loss_stats += ['off_loss'] if cfg.reg_obj: log_loss_stats += ['obj_loss'] for epoch in range(start_epoch + 1, num_epochs + 1): avg_loss_stats = {l: AverageMeter() for l in log_loss_stats} model.train() with tqdm(trainloader) as loader: for _, batch in enumerate(loader): for k in batch: if k != 'meta': batch[k] = batch[k].to(device=device, non_blocking=True) output, loss, loss_stats = model(batch) loss = loss.mean() optimizer.zero_grad() loss.backward() optimizer.step() # 设置tqdm显示信息 lr = optimizer.param_groups[0]['lr'] poststr = '' for l in avg_loss_stats: avg_loss_stats[l].update(loss_stats[l].mean().item(), batch['input'].size(0)) poststr += '{}: {:.4f}; '.format( l, avg_loss_stats[l].avg) loader.set_description('Epoch %d' % (epoch)) poststr += 'lr: {:.4f}'.format(lr) loader.set_postfix_str(poststr) step += 1 # self.lossSignal.emit(loss.item(), step) del output, loss, loss_stats # valid if step % val_step == 0: if len(cfg.gpus) > 1: val_model = model.module else: val_model = model val_model.eval() torch.cuda.empty_cache() # 随机采样 idx = np.arange(len(val_list)) idx = np.random.permutation(idx)[:sample_size] for j, id in enumerate(idx): image = cv2.imread(val_list[id]) image = self.preprocess(image, cfg.input_h, cfg.input_w, mean, std) image = image.to(device) with torch.no_grad(): output = val_model.model(image)[-1] # 画图并保存 debugger = Debugger(dataset=names, down_ratio=cfg.down_ratio) reg = output['reg'] if cfg.reg_offset else None obj = output['obj'] if cfg.reg_obj else None dets = ctdet_decode(output['hm'].sigmoid_(), output['wh'], reg=reg, obj=obj, cat_spec_wh=cfg.cat_spec_wh, K=cfg.K) dets = dets.detach().cpu().numpy().reshape( -1, dets.shape[2]) dets[:, :4] *= cfg.down_ratio image = image[0].detach().cpu().numpy().transpose( 1, 2, 0) image = np.clip(((image * std + mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][0].detach().cpu().numpy()) debugger.add_blend_img(image, pred, 'pred_hm') debugger.add_img(image, img_id='out_pred') for k in range(len(dets)): if dets[k, 4] > cfg.vis_thresh: debugger.add_coco_bbox(dets[k, :4], dets[k, -1], dets[k, 4], img_id='out_pred') debugger.save_all_imgs(debug_dir, prefix='{}.{}_'.format( step, j)) del output, image, dets # 保存模型参数 save_model(os.path.join(model_dir, 'model_best.pth'), epoch, net) model.train() logstr = 'epoch {}'.format(epoch) for k, v in avg_loss_stats.items(): logstr += ' {}: {:.4f};'.format(k, v.avg) if USE_TENSORBOARD: writer.add_scalar('train_{}'.format(k), v.avg, epoch) logger.info(logstr) # if epoch % val_step == 0: # if len(cfg.gpus) > 1: # val_model = model.module # else: # val_model = model # val_model.eval() # torch.cuda.empty_cache() # # val_loss_stats = {l: AverageMeter() for l in log_loss_stats} # # with tqdm(valloader) as loader: # for j, batch in enumerate(loader): # for k in batch: # if k != 'meta': # batch[k] = batch[k].to(device=device, non_blocking=True) # with torch.no_grad(): # output, loss, loss_stats = val_model(batch) # # poststr = '' # for l in val_loss_stats: # val_loss_stats[l].update( # loss_stats[l].mean().item(), batch['input'].size(0)) # poststr += '{}: {:.4f}; '.format(l, val_loss_stats[l].avg) # loader.set_description('Epoch %d valid' % (epoch)) # poststr += 'lr: {:.4f}'.format(lr) # loader.set_postfix_str(poststr) # # if j < sample_size: # # 将预测结果画出来保存成jpg图片 # debugger = Debugger(dataset=names, down_ratio=cfg.down_ratio) # reg = output['reg'] if cfg.reg_offset else None # obj = output['obj'] if cfg.reg_obj else None # dets = ctdet_decode( # output['hm'], output['wh'], reg=reg, obj=obj, # cat_spec_wh=cfg.cat_spec_wh, K=cfg.K) # dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) # dets[:, :, :4] *= cfg.down_ratio # dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) # dets_gt[:, :, :4] *= cfg.down_ratio # for i in range(1): # img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) # img = np.clip(((img * std + mean) * 255.), 0, 255).astype(np.uint8) # pred = debugger.gen_colormap(output['hm'][i].detach().cpu().numpy()) # gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) # debugger.add_blend_img(img, pred, 'pred_hm') # debugger.add_blend_img(img, gt, 'gt_hm') # debugger.add_img(img, img_id='out_pred') # for k in range(len(dets[i])): # if dets[i, k, 4] > cfg.vis_thresh: # debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], # dets[i, k, 4], img_id='out_pred') # # debugger.add_img(img, img_id='out_gt') # for k in range(len(dets_gt[i])): # if dets_gt[i, k, 4] > cfg.vis_thresh: # debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], # dets_gt[i, k, 4], img_id='out_gt') # # debugger.save_all_imgs(debug_dir, prefix='{}.{}_'.format(epoch, j)) # del output, loss, loss_stats # model.train() # logstr = 'epoch {} valid'.format(epoch) # for k, v in val_loss_stats.items(): # logstr += ' {}: {:.4f};'.format(k, v.avg) # if USE_TENSORBOARD: # writer.add_scalar('val_{}'.format(k), v.avg, epoch) # logger.info(logstr) # if val_loss_stats['loss'].avg < best: # best = val_loss_stats['loss'].avg # save_model(os.path.join(model_dir, 'model_best.pth'), epoch, net) save_model(os.path.join(model_dir, 'model_last.pth'), epoch, net, optimizer) if epoch in cfg.lr_step: save_model( os.path.join(model_dir, 'model_{}.pth'.format(epoch)), epoch, net, optimizer) lr = cfg.lr * (0.1**(cfg.lr_step.index(epoch) + 1)) logger.info('Drop LR to {}'.format(lr)) for param_group in optimizer.param_groups: param_group['lr'] = lr
import numpy as np from opts import opts from datasets.dataset.yolo import YOLO from utils.debugger import Debugger if __name__ == '__main__': opt = opts().parse() dataset = YOLO(opt.data_dir, opt.flip, opt.vflip, opt.rotate, opt.scale, opt.shear, opt, 'train') opt = opts().update_dataset_info_and_set_heads(opt, dataset) for i in range(len(dataset)): debugger = Debugger(dataset=opt.names) data = dataset[i] img = data['input'].transpose(1, 2, 0) hm = data['hm'] dets_gt = data['meta']['gt_det'] dets_gt[:, :4] *= opt.down_ratio img = np.clip(((img * dataset.std + dataset.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap(hm) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_img(img, img_id='out_pred') for k in range(len(dets_gt)): debugger.add_coco_bbox(dets_gt[k, :4], dets_gt[k, -1], dets_gt[k, 4], img_id='out_pred') debugger.show_all_imgs(pause=True)
def demo(opt): # creat folder save results os.mkdir('../Detection/bboxes_{}'.format(opt.arch)) os.mkdir('../visualization/{}'.format(opt.arch)) ### class_map = {1: 1, 2: 2} # color for boundingbox ### os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str ### opt.debug = max(opt.debug, 1) ### Detector = detector_factory[opt.task] detector = Detector(opt) assert os.path.isdir(opt.demo), 'Need path to videos directory.' video_paths = [ os.path.join(opt.demo, video_name) for video_name in os.listdir(opt.demo) if video_name.split('.')[-1] == 'mp4' ] # video_paths = [ # os.path.join(opt.demo, 'cam_2.mp4') # ] ### debugger = Debugger(dataset=opt.dataset, theme=opt.debugger_theme) ### for video_path in sorted(video_paths): bboxes = [] video = cv2.VideoCapture(video_path) width, height = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)), int( video.get(cv2.CAP_PROP_FRAME_HEIGHT)) ### bbox_video = cv2.VideoWriter( filename='../visualization/{}/'.format(opt.arch) + os.path.basename(video_path), fourcc=cv2.VideoWriter_fourcc(*'mp4v'), fps=float(30), frameSize=(width, height), isColor=True) ### num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) for i in tqdm(range(num_frames)): # skip_frame if opt.skip_frame > 0: if i % opt.skip_frame == 0: continue _, img = video.read() ret = detector.run(img) bboxes.append(ret['results']) ### debugger.add_img(img, img_id='default') for class_id in class_map.keys(): for bbox in ret['results'][class_id]: if bbox[4] > opt.vis_thresh: debugger.add_coco_bbox(bbox[:4], class_map[class_id], bbox[4], img_id='default') bbox_img = debugger.imgs['default'] bbox_video.write(bbox_img) ### with open( '../Detection/bboxes_{}'.format(opt.arch) + '/' + os.path.basename(video_path) + '.pkl', 'wb') as f: pickle.dump(bboxes, f)
def debug(self, batch, output, iter_id, dataset): opt = self.opt if "pre_hm" in batch: output.update({"pre_hm": batch["pre_hm"]}) dets = generic_decode(output, K=opt.K, opt=opt) for k in dets: dets[k] = dets[k].detach().cpu().numpy() dets_gt = batch["meta"]["gt_det"] for i in range(1): debugger = Debugger(opt=opt, dataset=dataset) img = batch["image"][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * dataset.std + dataset.mean) * 255.0), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output["hm"][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch["hm"][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, "pred_hm") debugger.add_blend_img(img, gt, "gt_hm") if "pre_img" in batch: pre_img = batch["pre_img"][i].detach().cpu().numpy().transpose( 1, 2, 0) pre_img = np.clip( ((pre_img * dataset.std + dataset.mean) * 255), 0, 255).astype(np.uint8) debugger.add_img(pre_img, "pre_img_pred") debugger.add_img(pre_img, "pre_img_gt") if "pre_hm" in batch: pre_hm = debugger.gen_colormap( batch["pre_hm"][i].detach().cpu().numpy()) debugger.add_blend_img(pre_img, pre_hm, "pre_hm") debugger.add_img(img, img_id="out_pred") if "ltrb_amodal" in opt.heads: debugger.add_img(img, img_id="out_pred_amodal") debugger.add_img(img, img_id="out_gt_amodal") # Predictions for k in range(len(dets["scores"][i])): if dets["scores"][i, k] > opt.vis_thresh: debugger.add_coco_bbox( dets["bboxes"][i, k] * opt.down_ratio, dets["clses"][i, k], dets["scores"][i, k], img_id="out_pred", ) if "ltrb_amodal" in opt.heads: debugger.add_coco_bbox( dets["bboxes_amodal"][i, k] * opt.down_ratio, dets["clses"][i, k], dets["scores"][i, k], img_id="out_pred_amodal", ) if "hps" in opt.heads and int(dets["clses"][i, k]) == 0: debugger.add_coco_hp(dets["hps"][i, k] * opt.down_ratio, img_id="out_pred") if "tracking" in opt.heads: debugger.add_arrow( dets["cts"][i][k] * opt.down_ratio, dets["tracking"][i][k] * opt.down_ratio, img_id="out_pred", ) debugger.add_arrow( dets["cts"][i][k] * opt.down_ratio, dets["tracking"][i][k] * opt.down_ratio, img_id="pre_img_pred", ) # Ground truth debugger.add_img(img, img_id="out_gt") for k in range(len(dets_gt["scores"][i])): if dets_gt["scores"][i][k] > opt.vis_thresh: debugger.add_coco_bbox( dets_gt["bboxes"][i][k] * opt.down_ratio, dets_gt["clses"][i][k], dets_gt["scores"][i][k], img_id="out_gt", ) if "ltrb_amodal" in opt.heads: debugger.add_coco_bbox( dets_gt["bboxes_amodal"][i, k] * opt.down_ratio, dets_gt["clses"][i, k], dets_gt["scores"][i, k], img_id="out_gt_amodal", ) if "hps" in opt.heads and (int(dets["clses"][i, k]) == 0): debugger.add_coco_hp(dets_gt["hps"][i][k] * opt.down_ratio, img_id="out_gt") if "tracking" in opt.heads: debugger.add_arrow( dets_gt["cts"][i][k] * opt.down_ratio, dets_gt["tracking"][i][k] * opt.down_ratio, img_id="out_gt", ) debugger.add_arrow( dets_gt["cts"][i][k] * opt.down_ratio, dets_gt["tracking"][i][k] * opt.down_ratio, img_id="pre_img_gt", ) if "hm_hp" in opt.heads: pred = debugger.gen_colormap_hp( output["hm_hp"][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp( batch["hm_hp"][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, "pred_hmhp") debugger.add_blend_img(img, gt, "gt_hmhp") if "rot" in opt.heads and "dim" in opt.heads and "dep" in opt.heads: dets_gt = {k: dets_gt[k].cpu().numpy() for k in dets_gt} calib = (batch["meta"]["calib"].detach().numpy() if "calib" in batch["meta"] else None) det_pred = generic_post_process( opt, dets, batch["meta"]["c"].cpu().numpy(), batch["meta"]["s"].cpu().numpy(), output["hm"].shape[2], output["hm"].shape[3], self.opt.num_classes, calib, ) det_gt = generic_post_process( opt, dets_gt, batch["meta"]["c"].cpu().numpy(), batch["meta"]["s"].cpu().numpy(), output["hm"].shape[2], output["hm"].shape[3], self.opt.num_classes, calib, ) debugger.add_3d_detection( batch["meta"]["img_path"][i], batch["meta"]["flipped"][i], det_pred[i], calib[i], vis_thresh=opt.vis_thresh, img_id="add_pred", ) debugger.add_3d_detection( batch["meta"]["img_path"][i], batch["meta"]["flipped"][i], det_gt[i], calib[i], vis_thresh=opt.vis_thresh, img_id="add_gt", ) debugger.add_bird_views( det_pred[i], det_gt[i], vis_thresh=opt.vis_thresh, img_id="bird_pred_gt", ) if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix="{}".format(iter_id)) else: debugger.show_all_imgs(pause=True)
def run(self, image_or_path_or_tensor, meta=None): load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0 merge_time, tot_time = 0, 0 debugger = Debugger(dataset=self.opt.dataset, ipynb=(self.opt.debug == 3), theme=self.opt.debugger_theme) start_time = time.time() pre_processed = False if isinstance(image_or_path_or_tensor, np.ndarray): image = image_or_path_or_tensor elif type(image_or_path_or_tensor) == type(''): image = cv2.imread(image_or_path_or_tensor) else: image = image_or_path_or_tensor['image'][0].numpy() pre_processed_images = image_or_path_or_tensor pre_processed = True loaded_time = time.time() load_time += (loaded_time - start_time) detections = [] for scale in self.scales: scale_start_time = time.time() if not pre_processed: images, meta = self.pre_process(image, scale, meta) else: # import pdb; pdb.set_trace() images = pre_processed_images['images'][scale][0] meta = pre_processed_images['meta'][scale] meta = {k: v.numpy()[0] for k, v in meta.items()} images = images.to(self.opt.device) torch.cuda.synchronize() pre_process_time = time.time() pre_time += pre_process_time - scale_start_time output, dets, forward_time = self.process(images, return_time=True) torch.cuda.synchronize() net_time += forward_time - pre_process_time decode_time = time.time() dec_time += decode_time - forward_time if self.opt.debug >= 2: self.debug(debugger, images, dets, output, scale) dets = self.post_process(dets, meta, scale) torch.cuda.synchronize() post_process_time = time.time() post_time += post_process_time - decode_time detections.append(dets) results = self.merge_outputs(detections) torch.cuda.synchronize() end_time = time.time() merge_time += end_time - post_process_time tot_time += end_time - start_time if self.opt.debug >= 1: # print('--->>> base_detector run show_results') # img_ = self.show_results(debugger, image, results) debugger.add_img(image, img_id='multi_pose') #---------------------------------------------------------------- NMS nms_dets_ = [] for bbox in results[1]: if bbox[4] > self.opt.vis_thresh: nms_dets_.append( (bbox[0], bbox[1], bbox[2], bbox[3], bbox[4])) if len(nms_dets_) > 0: keep_ = py_cpu_nms(np.array(nms_dets_), thresh=0.35) # print('keep_ : ',nms_dets_,keep_) #---------------------------------------------------------------- faces_boxes = [] person_boxes = [] idx = 0 for bbox in results[1]: if bbox[4] > self.opt.vis_thresh: idx += 1 if (idx - 1) not in keep_: continue # 绘制目标物体 # print('------------------>>>add_coco_bbox') debugger.add_coco_bbox(bbox[:4], 0, bbox[4], img_id='multi_pose') face_pts = debugger.add_coco_hp(bbox[5:39], img_id='multi_pose') # print('--------------------------------->>>>>>>>>>oou') if len(face_pts) == 5: # print('change box') person_boxes.append([ int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3]), bbox[4] ]) x_min = min( [face_pts[i][0] for i in range(len(face_pts))]) y_min = min( [face_pts[i][1] for i in range(len(face_pts))]) x_max = max( [face_pts[i][0] for i in range(len(face_pts))]) y_max = max( [face_pts[i][1] for i in range(len(face_pts))]) edge = abs(x_max - x_min) # bbox_x1 = int(max(0, (x_min - edge * 0.05))) bbox_x2 = int( min(image.shape[1] - 1, (x_max + edge * 0.05))) bbox_y1 = int(max(0, (y_min - edge * 0.32))) bbox_y2 = int( min(image.shape[0] - 1, (y_max + edge * 0.55))) # print('ppppp',face_pts,x1) # if ((bbox_x2-bbox_x1)*(bbox_y2-bbox_y1))>100: faces_boxes.append( [bbox_x1, bbox_y1, bbox_x2, bbox_y2, 1.]) # cv2.rectangle(image,(bbox_x1,bbox_y1),(bbox_x2,bbox_y2),(0,255,255),2) # print('-------->>> show_results debugger') img_ = debugger.show_all_imgs(pause=self.pause) return img_, { 'results': results, 'tot': tot_time, 'load': load_time, 'pre': pre_time, 'net': net_time, 'dec': dec_time, 'post': post_time, 'merge': merge_time }, faces_boxes, person_boxes
def run(self, image_or_path_or_tensor, meta=None): load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0 merge_time, tot_time = 0, 0 debugger = Debugger(dataset=self.opt.dataset, ipynb=(self.opt.debug==3), theme=self.opt.debugger_theme) start_time = time.time() pre_processed = False if isinstance(image_or_path_or_tensor, np.ndarray): image = image_or_path_or_tensor elif type(image_or_path_or_tensor) == type (''): image = cv2.imread(image_or_path_or_tensor) else: image = image_or_path_or_tensor['image'][0].numpy() pre_processed_images = image_or_path_or_tensor pre_processed = True loaded_time = time.time() load_time += (loaded_time - start_time) detections = [] for scale in self.scales: scale_start_time = time.time() if not pre_processed: images, meta = self.pre_process(image, scale, meta) else: # import pdb; pdb.set_trace() images = pre_processed_images['images'][scale][0] meta = pre_processed_images['meta'][scale] meta = {k: v.numpy()[0] for k, v in meta.items()} images = images.to(self.opt.device) torch.cuda.synchronize() pre_process_time = time.time() pre_time += pre_process_time - scale_start_time output, dets, forward_time = self.process(images, return_time=True) torch.cuda.synchronize() net_time += forward_time - pre_process_time decode_time = time.time() dec_time += decode_time - forward_time if self.opt.debug >= 2: self.debug(debugger, images, dets, output, scale) dets = self.post_process(dets, meta, scale) torch.cuda.synchronize() post_process_time = time.time() post_time += post_process_time - decode_time detections.append(dets) results = self.merge_outputs(detections) torch.cuda.synchronize() end_time = time.time() merge_time += end_time - post_process_time tot_time += end_time - start_time if self.opt.debug >= 1: # print('--->>> base_detector run show_results') # img_ = self.show_results(debugger, image, results) debugger.add_img(image, img_id='multi_pose') for bbox in results[1]: if bbox[4] > self.opt.vis_thresh: # 绘制目标物体 # print('------------------>>>add_coco_bbox') debugger.add_coco_bbox(bbox[:4], 0, bbox[4], img_id='multi_pose') debugger.add_coco_hp(bbox[5:39], img_id='multi_pose') # print('-------->>> show_results debugger') img_ = debugger.show_all_imgs(pause=self.pause) return img_,{'results': results, 'tot': tot_time, 'load': load_time, 'pre': pre_time, 'net': net_time, 'dec': dec_time, 'post': post_time, 'merge': merge_time}
def debug(self, batch, output, iter_id): opt = self.opt # reg = output['reg'] if opt.reg_offset else None reg = output['reg'][0:1] if opt.reg_offset else None # dets = ctdet_decode( # output['hm'], output['wh'], reg=reg, # cat_spec_wh=opt.cat_spec_wh, K=opt.K) dets = ctdet_decode(output['hm'][0:1], output['wh'][0:1], reg=reg, cat_spec_wh=opt.cat_spec_wh, K=opt.K) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= opt.down_ratio # FIXME: change from tensor to list and then reshape # dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) # batch['meta_gt_det'] = [128, 128, 6] gt_det = batch['meta_gt_det'][0:1] gt_det = np.array(gt_det, dtype=np.float32) if len(gt_det) > 0 else \ np.zeros((1, 6), dtype=np.float32) dets_gt = gt_det.reshape(1, -1, dets.shape[2]) # print(batch['meta_img_id'][0:1]) dets_gt[:, :, :4] *= opt.down_ratio for i in range(1): debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') for k in range(len(dets[i])): if dets[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) elif opt.debug == 5: debugger.show_all_imgs(pause=opt.pause, logger=self.logger, step=iter_id) else: debugger.show_all_imgs(pause=opt.pause, step=iter_id)