def debug(self, batch, output, iter_id): opt = self.opt reg = output['reg'] if opt.reg_offset else None hm_hp = output['hm_hp'] if opt.hm_hp else None hp_offset = output['hp_offset'] if opt.reg_hp_offset else None dets = multi_pose_decode(output['hm'], output['wh'], output['hps'], reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=opt.K) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= opt.input_res / opt.output_res dets[:, :, 5:39] *= opt.input_res / opt.output_res dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) dets_gt[:, :, :4] *= opt.input_res / opt.output_res dets_gt[:, :, 5:39] *= opt.input_res / opt.output_res for i in range(1): debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') for k in range(len(dets[i])): if dets[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_coco_hp(dets[i, k, 5:39], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.center_thresh: debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') debugger.add_coco_hp(dets_gt[i, k, 5:39], img_id='out_gt') if opt.hm_hp: pred = debugger.gen_colormap_hp( output['hm_hp'][i].detach().cpu().numpy()) gt = debugger.gen_colormap_hp( batch['hm_hp'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hmhp') debugger.add_blend_img(img, gt, 'gt_hmhp') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id): opt = self.opt reg = output['reg'] if opt.reg_offset else None dets = ctdet_decode(output['hm'], output['wh'], reg=reg, cat_spec_wh=opt.cat_spec_wh, K=opt.K) dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) dets[:, :, :4] *= opt.down_ratio dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2]) dets_gt[:, :, :4] *= opt.down_ratio for i in range(1): debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = np.clip(((img * opt.std + opt.mean) * 255.), 0, 255).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'pred_hm') debugger.add_blend_img(img, gt, 'gt_hm') debugger.add_img(img, img_id='out_pred') for k in range(len(dets[i])): if dets[i, k, 4] > opt.center_thresh: # add_coco_bbox(bbox, cat, conf=1, show_txt=True, img_id='default'): debugger.add_coco_bbox(dets[i, k, :4], dets[i, k, -1], dets[i, k, 4], img_id='out_pred') debugger.add_img(img, img_id='out_gt') for k in range(len(dets_gt[i])): if dets_gt[i, k, 4] > opt.center_thresh: # add_coco_bbox(bbox, cat, conf=1, show_txt=True, img_id='default'): debugger.add_coco_bbox(dets_gt[i, k, :4], dets_gt[i, k, -1], dets_gt[i, k, 4], img_id='out_gt') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id): opt = self.opt wh = output['wh'] if opt.reg_bbox else None reg = output['reg'] if opt.reg_offset else None dets = ddd_decode(output['hm'], output['rot'], output['dep'], output['dim'], wh=wh, reg=reg, K=opt.K) # x, y, score, r1-r8, depth, dim1-dim3, cls dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2]) calib = batch['meta']['calib'].detach().numpy() # x, y, score, rot, depth, dim1, dim2, dim3 # if opt.dataset == 'gta': # dets[:, 12:15] /= 3 dets_pred = ddd_post_process( dets.copy(), batch['meta']['c'].detach().numpy(), batch['meta']['s'].detach().numpy(), calib, opt) dets_gt = ddd_post_process( batch['meta']['gt_det'].detach().numpy().copy(), batch['meta']['c'].detach().numpy(), batch['meta']['s'].detach().numpy(), calib, opt) #for i in range(input.size(0)): for i in range(1): debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug==3), theme=opt.debugger_theme) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = ((img * self.opt.std + self.opt.mean) * 255.).astype(np.uint8) pred = debugger.gen_colormap( output['hm'][i].detach().cpu().numpy()) gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy()) debugger.add_blend_img(img, pred, 'hm_pred') debugger.add_blend_img(img, gt, 'hm_gt') # decode debugger.add_ct_detection( img, dets[i], show_box=opt.reg_bbox, center_thresh=opt.center_thresh, img_id='det_pred') debugger.add_ct_detection( img, batch['meta']['gt_det'][i].cpu().numpy().copy(), show_box=opt.reg_bbox, img_id='det_gt') debugger.add_3d_detection( batch['meta']['image_path'][i], dets_pred[i], calib[i], center_thresh=opt.center_thresh, img_id='add_pred') debugger.add_3d_detection( batch['meta']['image_path'][i], dets_gt[i], calib[i], center_thresh=opt.center_thresh, img_id='add_gt') # debugger.add_bird_view( # dets_pred[i], center_thresh=opt.center_thresh, img_id='bird_pred') # debugger.add_bird_view(dets_gt[i], img_id='bird_gt') debugger.add_bird_views( dets_pred[i], dets_gt[i], center_thresh=opt.center_thresh, img_id='bird_pred_gt') # debugger.add_blend_img(img, pred, 'out', white=True) debugger.compose_vis_add( batch['meta']['image_path'][i], dets_pred[i], calib[i], opt.center_thresh, pred, 'bird_pred_gt', img_id='out') # debugger.add_img(img, img_id='out') if opt.debug ==4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)
def run(self, image_or_path_or_tensor, meta=None): load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0 merge_time, tot_time = 0, 0 debugger = Debugger(dataset=self.opt.dataset, ipynb=(self.opt.debug==3), theme=self.opt.debugger_theme) start_time = time.time() pre_processed = False if isinstance(image_or_path_or_tensor, np.ndarray): image = image_or_path_or_tensor elif type(image_or_path_or_tensor) == type (''): image = cv2.imread(image_or_path_or_tensor) else: image = image_or_path_or_tensor['image'][0].numpy() pre_image = image_or_path_or_tensor['pre_img'][0].numpy() pre_processed_images = image_or_path_or_tensor pre_processed = True loaded_time = time.time() load_time += (loaded_time - start_time) detections = [] for scale in self.scales: scale_start_time = time.time() if not pre_processed: images, meta = self.pre_process(image, scale, meta) else: # import pdb; pdb.set_trace() images = pre_processed_images['images'][scale][0] pre_images = pre_processed_images['pre_images'][scale][0] meta = pre_processed_images['meta'][scale] meta = {k: v.numpy()[0] for k, v in meta.items()} images = images.to(self.opt.device) pre_images = pre_images.to(self.opt.device) torch.cuda.synchronize() pre_process_time = time.time() pre_time += pre_process_time - scale_start_time output, dets, forward_time = self.process(images, pre_images, return_time=True) torch.cuda.synchronize() net_time += forward_time - pre_process_time decode_time = time.time() dec_time += decode_time - forward_time if self.opt.debug >= 2: self.debug(debugger, images, dets, output, scale) dets = self.post_process(dets, meta, scale) torch.cuda.synchronize() post_process_time = time.time() post_time += post_process_time - decode_time detections.append(dets) results = self.merge_outputs(detections) torch.cuda.synchronize() end_time = time.time() merge_time += end_time - post_process_time tot_time += end_time - start_time if self.opt.debug >= 1: self.show_results(debugger, image, results) return {'results': results, 'tot': tot_time, 'load': load_time, 'pre': pre_time, 'net': net_time, 'dec': dec_time, 'post': post_time, 'merge': merge_time}
def debug(self, batch, output, iter_id): opt = self.opt detections = self.decode(output['hm_t'], output['hm_l'], output['hm_b'], output['hm_r'], output['hm_c']).detach().cpu().numpy() detections[:, :, :4] *= opt.input_res / opt.output_res for i in range(1): debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug == 3), theme=opt.debugger_theme) pred_hm = np.zeros((opt.input_res, opt.input_res, 3), dtype=np.uint8) gt_hm = np.zeros((opt.input_res, opt.input_res, 3), dtype=np.uint8) img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0) img = ((img * self.opt.std + self.opt.mean) * 255.).astype( np.uint8) for p in self.parts: tag = 'hm_{}'.format(p) pred = debugger.gen_colormap( output[tag][i].detach().cpu().numpy()) gt = debugger.gen_colormap( batch[tag][i].detach().cpu().numpy()) if p != 'c': pred_hm = np.maximum(pred_hm, pred) gt_hm = np.maximum(gt_hm, gt) if p == 'c' or opt.debug > 2: debugger.add_blend_img(img, pred, 'pred_{}'.format(p)) debugger.add_blend_img(img, gt, 'gt_{}'.format(p)) debugger.add_blend_img(img, pred_hm, 'pred') debugger.add_blend_img(img, gt_hm, 'gt') debugger.add_img(img, img_id='out') for k in range(len(detections[i])): if detections[i, k, 4] > 0.1: debugger.add_coco_bbox(detections[i, k, :4], detections[i, k, -1], detections[i, k, 4], img_id='out') if opt.debug == 4: debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id)) else: debugger.show_all_imgs(pause=True)