def multi_gpu_test_net_on_dataset(num_images, output_dir): binary = os.path.join('tools/test_net.py') assert os.path.exists(binary), 'Binary {} not found'.format(binary) # Run inference in parallel in subprocesses outputs = subprocess_utils.process_in_parallel('detection', num_images, binary, output_dir) # Collate the results from each subprocess all_boxes = [[] for _ in range(cfg.MODEL.NUM_CLASSES)] all_segms = [[] for _ in range(cfg.MODEL.NUM_CLASSES)] all_keyps = [[] for _ in range(cfg.MODEL.NUM_CLASSES)] for det_data in outputs: all_boxes_batch = det_data['all_boxes'] all_segms_batch = det_data['all_segms'] all_keyps_batch = det_data['all_keyps'] for j in range(1, cfg.MODEL.NUM_CLASSES): all_boxes[j] += all_boxes_batch[j] all_segms[j] += all_segms_batch[j] all_keyps[j] += all_keyps_batch[j] det_file = os.path.join(output_dir, 'detections.pkl') cfg_yaml = yaml.dump(cfg) robust_pickle_dump( dict(all_boxes=all_boxes, all_segms=all_segms, all_keyps=all_keyps, cfg=cfg_yaml), det_file) logger.info('Wrote detections to: {}'.format(os.path.abspath(det_file))) return all_boxes, all_segms, all_keyps
def multi_gpu_test_net_on_dataset(num_images, output_dir): binary = os.path.join('tools/test_net.py') assert os.path.exists(binary), 'Binary {} not found'.format(binary) # Run inference in parallel in subprocesses outputs = subprocess_utils.process_in_parallel( 'detection', num_images, binary, output_dir) # Collate the results from each subprocess all_boxes = [[] for _ in range(cfg.MODEL.NUM_CLASSES)] all_segms = [[] for _ in range(cfg.MODEL.NUM_CLASSES)] all_keyps = [[] for _ in range(cfg.MODEL.NUM_CLASSES)] for det_data in outputs: all_boxes_batch = det_data['all_boxes'] all_segms_batch = det_data['all_segms'] all_keyps_batch = det_data['all_keyps'] for j in range(1, cfg.MODEL.NUM_CLASSES): all_boxes[j] += all_boxes_batch[j] all_segms[j] += all_segms_batch[j] all_keyps[j] += all_keyps_batch[j] det_file = os.path.join(output_dir, 'detections.pkl') cfg_yaml = yaml.dump(cfg) robust_pickle_dump( dict(all_boxes=all_boxes, all_segms=all_segms, all_keyps=all_keyps, cfg=cfg_yaml), det_file) logger.info('Wrote detections to: {}'.format(os.path.abspath(det_file))) return all_boxes, all_segms, all_keyps
def main(name_scope, gpu_dev, num_images, args): t=args.t model = initialize_model_from_cfg() num_classes = cfg.MODEL.NUM_CLASSES all_boxes, all_segms, all_keyps = empty_results(num_classes, num_images) temp_frame_folder = osp.join(args.out_path,args.vid_name + '_frames/',str(t)) imgs = glob.glob(temp_frame_folder+'/*.jpg') for i in range(len(imgs)): if i%100==0: print('Processing Detection for Frame %d'%(i+1)) im_ = cv2.imread(imgs[i]) assert im_ is not None im_ = np.expand_dims(im_, 0) with core.NameScope(name_scope): with core.DeviceScope(gpu_dev): cls_boxes_i, cls_segms_i, cls_keyps_i = im_detect_all( model, im_, None) #TODO: Parallelize detection extend_results(i, all_boxes, cls_boxes_i) if cls_segms_i is not None: extend_results(i, all_segms, cls_segms_i) if cls_keyps_i is not None: extend_results(i, all_keyps, cls_keyps_i) det_name = args.vid_name + '_' + str(args.t) + '_detections.pkl' det_file = osp.join(args.out_path, det_name) robust_pickle_dump(dict(all_keyps=all_keyps),det_file) shutil.rmtree(osp.join(args.out_path, args.vid_name + '_frames'))
def generate_rpn_on_range(ind_range=None): assert cfg.TEST.WEIGHTS != '', \ 'TEST.WEIGHTS must be set to the model file to test' assert cfg.TEST.DATASET != '', \ 'TEST.DATASET must be set to the dataset name to test' assert cfg.MODEL.RPN_ONLY or cfg.MODEL.FASTER_RCNN im_list, start_ind, end_ind, total_num_images = get_image_list(ind_range) output_dir = get_output_dir(training=False) logger.info( 'Output will be saved to: {:s}'.format(os.path.abspath(output_dir))) model = model_builder.create(cfg.MODEL.TYPE, train=False) model_builder.add_inputs(model) nu.initialize_from_weights_file(model, cfg.TEST.WEIGHTS) workspace.CreateNet(model.net) boxes, scores, ids = im_list_proposals( model, im_list, start_ind=start_ind, end_ind=end_ind, total_num_images=total_num_images) cfg_yaml = yaml.dump(cfg) if ind_range is not None: rpn_name = 'rpn_proposals_range_%s_%s.pkl' % tuple(ind_range) else: rpn_name = 'rpn_proposals.pkl' rpn_file = os.path.join(output_dir, rpn_name) robust_pickle_dump( dict(boxes=boxes, scores=scores, ids=ids, cfg=cfg_yaml), rpn_file) logger.info('Wrote RPN proposals to {}'.format(os.path.abspath(rpn_file))) return boxes, scores, ids, rpn_file
def multi_gpu_generate_rpn_on_dataset(num_images, output_dir): # TODO(rbg): Need to have non-FB specific code path for OSS if cfg.CLUSTER.ON_CLUSTER: binary_dir = os.path.abspath(os.getcwd()) binary = os.path.join(binary_dir, 'test_net.xar') else: assert parutil.is_lpar(), 'Binary must be inplace package style' binary_dir = os.path.dirname(parutil.get_runtime_path()) binary = os.path.join(binary_dir, 'test_net.par') assert os.path.exists(binary), 'Binary {} not found'.format(binary) # Run inference in parallel in subprocesses outputs = subprocess_utils.process_in_parallel( 'rpn_proposals', num_images, binary, output_dir) # Collate the results from each subprocess boxes, scores, ids = [], [], [] for rpn_data in outputs: boxes += rpn_data['boxes'] scores += rpn_data['scores'] ids += rpn_data['ids'] rpn_file = os.path.join(output_dir, 'rpn_proposals.pkl') cfg_yaml = yaml.dump(cfg) robust_pickle_dump( dict(boxes=boxes, scores=scores, ids=ids, cfg=cfg_yaml), rpn_file) logger.info('Wrote RPN proposals to {}'.format(os.path.abspath(rpn_file))) return boxes, scores, ids, rpn_file
def _do_detection_eval(json_dataset, res_file, output_dir): coco_dt = json_dataset.COCO.loadRes(str(res_file)) coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox') coco_eval.evaluate() coco_eval.accumulate() _log_detection_eval_metrics(json_dataset, coco_eval) eval_file = os.path.join(output_dir, 'detection_results.pkl') robust_pickle_dump(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file))
def _do_segmentation_eval(json_dataset, res_file, output_dir): coco_dt = json_dataset.COCO.loadRes(str(res_file)) coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'segm') coco_eval.evaluate() coco_eval.accumulate() _log_detection_eval_metrics(json_dataset, coco_eval) eval_file = os.path.join(output_dir, 'segmentation_results.pkl') robust_pickle_dump(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file))
def _do_keypoint_eval(json_dataset, res_file, output_dir): ann_type = 'keypoints' imgIds = json_dataset.COCO.getImgIds() imgIds.sort() coco_dt = json_dataset.COCO.loadRes(res_file) coco_eval = COCOeval(json_dataset.COCO, coco_dt, ann_type) coco_eval.params.imgIds = imgIds coco_eval.evaluate() coco_eval.accumulate() eval_file = os.path.join(output_dir, 'keypoint_results.pkl') robust_pickle_dump(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file)) coco_eval.summarize()
def _do_keypoint_eval(json_dataset, res_file, output_dir): ann_type = 'keypoints' imgIds = json_dataset.COCO.getImgIds() imgIds.sort() coco_dt = json_dataset.COCO.loadRes(res_file) coco_eval = COCOeval(json_dataset.COCO, coco_dt, ann_type) coco_eval.params.imgIds = imgIds coco_eval.evaluate() coco_eval.accumulate() eval_file = os.path.join(output_dir, 'keypoint_results.pkl') robust_pickle_dump(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file)) coco_eval.summarize()
def save_model_to_weights_file(weights_file, model): """Stash model weights in a dictionary and pickle them to a file. We map GPU device scoped names to unscoped names (e.g., 'gpu_0/conv1_w' -> 'conv1_w'). """ logger.info( 'Saving parameters and momentum to {}'.format( os.path.abspath(weights_file))) blobs = {} # Save all parameters for param in model.params: scoped_name = str(param) unscoped_name = utils.blob.unscope_name(scoped_name) if unscoped_name not in blobs: logger.debug(' {:s} -> {:s}'.format(scoped_name, unscoped_name)) blobs[unscoped_name] = workspace.FetchBlob(scoped_name) # Save momentum for param in model.TrainableParams(): scoped_name = str(param) + '_momentum' unscoped_name = utils.blob.unscope_name(scoped_name) if unscoped_name not in blobs: logger.debug(' {:s} -> {:s}'.format(scoped_name, unscoped_name)) blobs[unscoped_name] = workspace.FetchBlob(scoped_name) # Save preserved blobs for scoped_name in workspace.Blobs(): if scoped_name.startswith('__preserve__/'): unscoped_name = utils.blob.unscope_name(scoped_name) if unscoped_name not in blobs: logger.debug( ' {:s} -> {:s} (preserved)'.format( scoped_name, unscoped_name)) blobs[unscoped_name] = workspace.FetchBlob(scoped_name) # Save the _rm/_riv for the models with batch norm for scoped_name in workspace.Blobs(): if scoped_name.endswith('_rm') or scoped_name.endswith('_riv'): unscoped_name = utils.blob.unscope_name(scoped_name) if unscoped_name not in blobs: logger.debug( ' {:s} -> {:s} (preserved)'.format( scoped_name, unscoped_name)) blobs[unscoped_name] = workspace.FetchBlob(scoped_name) cfg_yaml = yaml.dump(cfg) robust_pickle_dump(dict(blobs=blobs, cfg=cfg_yaml), weights_file)
def evaluate_proposal_file(dataset, proposal_file, output_dir): roidb = dataset.get_roidb(gt=True, proposal_file=proposal_file) logger.info('~~~~ Summary metrics ~~~~') recall_files = [] for l in [300, 1000, 2000]: print(' res@{:d} proposals / image:'.format(l)) res = {} for a in ['all', 'small', 'medium', 'large']: res[a] = json_dataset_evaluator.evaluate_recall( dataset, roidb, area=a, limit=l) print(' area={:8s} | ar={:.3f}'.format(a, res[a]['ar'])) # Index 4 is for iou thresh of 0.7 for a in ['all', 'small', 'medium', 'large']: print( ' iou=[.7] | area={:8s} | ar={:.3f}'. format(a, res[a]['recalls'][4])) recall_file = os.path.join( output_dir, 'at{:d}'.format(l) + 'rpn_proposal_recall.pkl') robust_pickle_dump(res, recall_file) recall_files.append(recall_file) logger.info('Evaluating proposals is done!') return recall_files
def test_net(ind_range=None): assert cfg.TEST.WEIGHTS != '', \ 'TEST.WEIGHTS must be set to the model file to test' assert not cfg.MODEL.RPN_ONLY, \ 'Use rpn_generate to generate proposals from RPN-only models' assert cfg.TEST.DATASET != '', \ 'TEST.DATASET must be set to the dataset name to test' output_dir = get_output_dir(training=False) roidb, dataset, start_ind, end_ind, total_num_images = \ get_roidb_and_dataset(ind_range) model = initialize_model_from_cfg() num_images = len(roidb) num_classes = cfg.MODEL.NUM_CLASSES all_boxes, all_segms, all_keyps = empty_results(num_classes, num_images) timers = defaultdict(Timer) gpu_dev = core.DeviceOption(caffe2_pb2.CUDA, cfg.ROOT_GPU_ID) name_scope = 'gpu_{}'.format(cfg.ROOT_GPU_ID) for i, entry in enumerate(roidb): if cfg.MODEL.FASTER_RCNN: box_proposals = None else: # The roidb may contain ground-truth rois (for example, if the roidb # comes from the training or val split). We only want to evaluate # detection on the *non*-ground-truth rois. We select only the rois # that have the gt_classes field set to 0, which means there's no # ground truth. box_proposals = entry['boxes'][entry['gt_classes'] == 0] if len(box_proposals) == 0: continue im = image_utils.read_image_video(entry) with core.NameScope(name_scope): with core.DeviceScope(gpu_dev): cls_boxes_i, cls_segms_i, cls_keyps_i = im_detect_all( model, im, box_proposals, timers) extend_results(i, all_boxes, cls_boxes_i) if cls_segms_i is not None: extend_results(i, all_segms, cls_segms_i) if cls_keyps_i is not None: extend_results(i, all_keyps, cls_keyps_i) if i % 10 == 0: # Reduce log file size ave_total_time = np.sum([t.average_time for t in timers.values()]) eta_seconds = ave_total_time * (num_images - i - 1) eta = str(datetime.timedelta(seconds=int(eta_seconds))) det_time = (timers['im_detect_bbox'].average_time + timers['im_detect_mask'].average_time + timers['im_detect_keypoints'].average_time) misc_time = (timers['misc_bbox'].average_time + timers['misc_mask'].average_time + timers['misc_keypoints'].average_time) logger.info(('im_detect: range [{:d}, {:d}] of {:d}: ' '{:d}/{:d} {:.3f}s + {:.3f}s (eta: {})').format( start_ind + 1, end_ind, total_num_images, start_ind + i + 1, start_ind + num_images, det_time, misc_time, eta)) if cfg.VIS: im_name = os.path.splitext(os.path.basename(entry['image']))[0] vis_utils.vis_one_image(im[:, :, ::-1], '{:d}_{:s}'.format(i, im_name), os.path.join(output_dir, 'vis'), cls_boxes_i, segms=cls_segms_i, keypoints=cls_keyps_i, thresh=cfg.VIS_THR, box_alpha=0.8, dataset=dataset, show_class=True) cfg_yaml = yaml.dump(cfg) if ind_range is not None: det_name = 'detection_range_%s_%s.pkl' % tuple(ind_range) else: det_name = 'detections.pkl' det_file = os.path.join(output_dir, det_name) robust_pickle_dump( dict(all_boxes=all_boxes, all_segms=all_segms, all_keyps=all_keyps, cfg=cfg_yaml), det_file) logger.info('Wrote detections to: {}'.format(os.path.abspath(det_file))) return all_boxes, all_segms, all_keyps
def test_net(ind_range=None): assert cfg.TEST.WEIGHTS != '', \ 'TEST.WEIGHTS must be set to the model file to test' assert not cfg.MODEL.RPN_ONLY, \ 'Use rpn_generate to generate proposals from RPN-only models' assert cfg.TEST.DATASET != '', \ 'TEST.DATASET must be set to the dataset name to test' output_dir = get_output_dir(training=False) roidb, dataset, start_ind, end_ind, total_num_images = \ get_roidb_and_dataset(ind_range) model = initialize_model_from_cfg() num_images = len(roidb) num_classes = cfg.MODEL.NUM_CLASSES all_boxes, all_segms, all_keyps = empty_results(num_classes, num_images) timers = defaultdict(Timer) gpu_dev = core.DeviceOption(caffe2_pb2.CUDA, cfg.ROOT_GPU_ID) name_scope = 'gpu_{}'.format(cfg.ROOT_GPU_ID) for i, entry in enumerate(roidb): if cfg.MODEL.FASTER_RCNN: box_proposals = None else: # The roidb may contain ground-truth rois (for example, if the roidb # comes from the training or val split). We only want to evaluate # detection on the *non*-ground-truth rois. We select only the rois # that have the gt_classes field set to 0, which means there's no # ground truth. box_proposals = entry['boxes'][entry['gt_classes'] == 0] if len(box_proposals) == 0: continue im = image_utils.read_image_video(entry) with core.NameScope(name_scope): with core.DeviceScope(gpu_dev): cls_boxes_i, cls_segms_i, cls_keyps_i = im_detect_all( model, im, box_proposals, timers) extend_results(i, all_boxes, cls_boxes_i) if cls_segms_i is not None: extend_results(i, all_segms, cls_segms_i) if cls_keyps_i is not None: extend_results(i, all_keyps, cls_keyps_i) if i % 10 == 0: # Reduce log file size ave_total_time = np.sum([t.average_time for t in timers.values()]) eta_seconds = ave_total_time * (num_images - i - 1) eta = str(datetime.timedelta(seconds=int(eta_seconds))) det_time = (timers['im_detect_bbox'].average_time + timers['im_detect_mask'].average_time + timers['im_detect_keypoints'].average_time) misc_time = (timers['misc_bbox'].average_time + timers['misc_mask'].average_time + timers['misc_keypoints'].average_time) logger.info( ('im_detect: range [{:d}, {:d}] of {:d}: ' '{:d}/{:d} {:.3f}s + {:.3f}s (eta: {})').format( start_ind + 1, end_ind, total_num_images, start_ind + i + 1, start_ind + num_images, det_time, misc_time, eta)) if cfg.VIS: im_name = os.path.splitext(os.path.basename(entry['image']))[0] vis_utils.vis_one_image( im[:, :, ::-1], '{:d}_{:s}'.format(i, im_name), os.path.join(output_dir, 'vis'), cls_boxes_i, segms=cls_segms_i, keypoints=cls_keyps_i, thresh=cfg.VIS_THR, box_alpha=0.8, dataset=dataset, show_class=True) cfg_yaml = yaml.dump(cfg) if ind_range is not None: det_name = 'detection_range_%s_%s.pkl' % tuple(ind_range) else: det_name = 'detections.pkl' det_file = os.path.join(output_dir, det_name) robust_pickle_dump( dict(all_boxes=all_boxes, all_segms=all_segms, all_keyps=all_keyps, cfg=cfg_yaml), det_file) logger.info('Wrote detections to: {}'.format(os.path.abspath(det_file))) return all_boxes, all_segms, all_keyps
def main(name_scope, gpu_dev, num_images, args): model = initialize_model_from_cfg() num_classes = cfg.MODEL.NUM_CLASSES all_boxes, all_segms, all_keyps = empty_results(num_classes, num_images) if '2d_best' in args.cfg_file: for i in range(num_images): print('Processing Detection for Frame %d'%(i+1)) im_ = _read_video_frames(args.out_path, args.vid_name, i) im_ = np.expand_dims(im_, 0) with core.NameScope(name_scope): with core.DeviceScope(gpu_dev): cls_boxes_i, cls_segms_i, cls_keyps_i = im_detect_all( model, im_, None) #TODO: Parallelize detection extend_results(i, all_boxes, cls_boxes_i) if cls_segms_i is not None: extend_results(i, all_segms, cls_segms_i) if cls_keyps_i is not None: extend_results(i, all_keyps, cls_keyps_i) elif '3d' in args.cfg_file: for i in range(num_images-2): print('Processing Detection for Frame %d to Frame %d' % (i + 1, i + 2)) ims_ = _read_video_3frames(args.out_path, args.vid_name, i) # ims_ = np.expand_dims(ims_, 0) with core.NameScope(name_scope): with core.DeviceScope(gpu_dev): cls_boxes_i, cls_segms_i, cls_keyps_i = im_detect_all( model, ims_, None) # extend boxes for 3 frames tmp_boxes_i2 = deepcopy(cls_boxes_i) tmp_boxes_i2[1] = tmp_boxes_i2[1][:, 8:] extend_results(i+2, all_boxes, tmp_boxes_i2) tmp_boxes_i1 = deepcopy(cls_boxes_i) tmp_boxes_i1[1] = tmp_boxes_i1[1][:, [4, 5, 6, 7, -1]] extend_results(i+1, all_boxes, tmp_boxes_i1) tmp_boxes_i0 = deepcopy(cls_boxes_i) tmp_boxes_i0[1] = tmp_boxes_i0[1][:, [0, 1, 2, 3, -1]] extend_results(i, all_boxes, tmp_boxes_i0) # extend segms for 3 frames if cls_segms_i is not None: extend_results(i+2, all_segms, cls_segms_i) # extend keyps for 3 frames if cls_keyps_i is not None: # extend the i+2 th one tmp_keyps_i2 = deepcopy(cls_keyps_i) for idx in range(len(tmp_keyps_i2[1])): tmp_keyps_i2[1][idx] = tmp_keyps_i2[1][idx][:, 34:] extend_results(i+2, all_keyps, tmp_keyps_i2) # extend the i+1 th one tmp_keyps_i1 = deepcopy(cls_keyps_i) for idx in range(len(tmp_keyps_i1[1])): tmp_keyps_i1[1][idx] = tmp_keyps_i1[1][idx][:, 17:34] extend_results(i + 1, all_keyps, tmp_keyps_i1) # extend the i th one tmp_keyps_i0 = deepcopy(cls_keyps_i) for idx in range(len(tmp_keyps_i0[1])): tmp_keyps_i0[1][idx] = tmp_keyps_i0[1][idx][:, :17] extend_results(i, all_keyps, tmp_keyps_i0) cfg_yaml = yaml.dump(cfg) det_name = args.vid_name + '_detections.pkl' det_file = osp.join(args.out_path, det_name) robust_pickle_dump( dict(all_boxes=all_boxes, all_segms=all_segms, all_keyps=all_keyps, cfg=cfg_yaml), det_file) frames = sorted(glob.glob(osp.join(args.out_path,args.vid_name + '_frames','*.jpg'))) out_detrack_file = osp.join(args.out_path, args.vid_name + '_detections_withTracks.pkl') # Debug configurations if cfg.TRACKING.DEBUG.UPPER_BOUND_2_GT_KPS: # if this is true cfg.TRACKING.DEBUG.UPPER_BOUND = True # This must be set true # Set include_gt True when using the roidb to evalute directly. Not doing # that currently dets = _load_det_file(det_file) if cfg.TRACKING.KEEP_CENTER_DETS_ONLY: _center_detections(dets) conf = cfg.TRACKING.CONF_FILTER_INITIAL_DETS dets = _prune_bad_detections(dets, conf) if cfg.TRACKING.LSTM_TEST.LSTM_TRACKING_ON: # Needs torch, only importing if we need to run LSTM tracking from lstm.lstm_track import lstm_track_utils lstm_model = lstm_track_utils.init_lstm_model( cfg.TRACKING.LSTM_TEST.LSTM_WEIGHTS) lstm_model.cuda() else: lstm_model = None dets_withTracks = compute_matches_tracks(frames, dets, lstm_model) _write_det_file(dets_withTracks, out_detrack_file) for i in range(num_images): vis_im = _generate_visualizations(frames[i], i, dets['all_boxes'], dets['all_keyps'], dets['all_tracks']) cv2.imwrite(osp.join(args.out_path, args.vid_name + '_vis','%08d.jpg'%(i+1)),vis_im)