def evaluate(): cocoGt = COCO('annotations.json') cocoDt = cocoGt.loadRes('detections.json') cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def _do_python_eval(self, _coco): coco_dt = _coco.loadRes(self._result_file) coco_eval = COCOeval(_coco, coco_dt) coco_eval.params.useSegm = False coco_eval.evaluate() coco_eval.accumulate() self._print_detection_metrics(coco_eval)
def coco_evaluate(json_dataset, res_file, image_ids): coco_dt = json_dataset.COCO.loadRes(str(res_file)) coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval
def _do_segmentation_eval(json_dataset, res_file, output_dir): coco_dt = json_dataset.COCO.loadRes(str(res_file)) coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'segm') coco_eval.evaluate() coco_eval.accumulate() _log_detection_eval_metrics(json_dataset, coco_eval) eval_file = os.path.join(output_dir, 'segmentation_results.pkl') robust_pickle_dump(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file))
def _do_detection_eval(json_dataset, res_file, output_dir): coco_dt = json_dataset.COCO.loadRes(str(res_file)) coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox') coco_eval.evaluate() coco_eval.accumulate() _log_detection_eval_metrics(json_dataset, coco_eval) eval_file = os.path.join(output_dir, 'detection_results.pkl') save_object(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file)) return coco_eval
def cocoval(detected_json): eval_json = config.eval_json eval_gt = COCO(eval_json) eval_dt = eval_gt.loadRes(detected_json) cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox') # cocoEval.params.imgIds = eval_gt.getImgIds() cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def _do_coco_eval(self, dtFile, output_dir): """ Evaluate using COCO API """ if self._image_set == 'train' or self._image_set == 'val': cocoGt = self._coco[0] cocoDt = COCO(dtFile) E = COCOeval(cocoGt, cocoDt) E.evaluate() E.accumulate() E.summarize()
def evaluate_detections(self, all_boxes, output_dir=None): resFile = self._write_coco_results_file(all_boxes) cocoGt = self._annotations cocoDt = cocoGt.loadRes(resFile) # running evaluation cocoEval = COCOeval(cocoGt,cocoDt) # useSegm should default to 0 #cocoEval.params.useSegm = 0 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def compute_ap(self): coco_res = self.loader.coco.loadRes(self.filename) cocoEval = COCOeval(self.loader.coco, coco_res) cocoEval.params.imgIds = self.loader.get_filenames() cocoEval.params.useSegm = False cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() return cocoEval
def _do_detection_eval(self, res_file, output_dir): ann_type = 'bbox' coco_dt = self._COCO.loadRes(res_file) coco_eval = COCOeval(self._COCO, coco_dt) coco_eval.params.useSegm = (ann_type == 'segm') coco_eval.evaluate() coco_eval.accumulate() self._print_detection_eval_metrics(coco_eval) eval_file = osp.join(output_dir, 'detection_results.pkl') with open(eval_file, 'wb') as fid: pickle.dump(coco_eval, fid, pickle.HIGHEST_PROTOCOL) print('Wrote COCO eval results to: {}'.format(eval_file))
def _do_keypoint_eval(json_dataset, res_file, output_dir): ann_type = 'keypoints' imgIds = json_dataset.COCO.getImgIds() imgIds.sort() coco_dt = json_dataset.COCO.loadRes(res_file) coco_eval = COCOeval(json_dataset.COCO, coco_dt, ann_type) coco_eval.params.imgIds = imgIds coco_eval.evaluate() coco_eval.accumulate() eval_file = os.path.join(output_dir, 'keypoint_results.pkl') robust_pickle_dump(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file)) coco_eval.summarize()
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation limit: if not 0, it's the number of images to use for evaluation """ # Pick COCO images from the dataset image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] # Get corresponding COCO image IDs. coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): # Load image image = dataset.load_image(image_id) # Run detection t = time.time() r = model.detect([image], verbose=0)[0] t_prediction += (time.time() - t) # Convert results to COCO format # Cast masks to uint8 because COCO tools errors out on bool image_results = build_coco_results(dataset, coco_image_ids[i:i + 1], r["rois"], r["class_ids"], r["scores"], r["masks"].astype(np.uint8)) results.extend(image_results) # Load results. This modifies results with additional attributes. coco_results = coco.loadRes(results) # Evaluate cocoEval = COCOeval(coco, coco_results, eval_type) cocoEval.params.imgIds = coco_image_ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start)
def validate(val_loader, model, i, silence=False): batch_time = AverageMeter() coco_gt = val_loader.dataset.coco coco_pred = COCO() coco_pred.dataset['images'] = [img for img in coco_gt.datasets['images']] coco_pred.dataset['categories'] = copy.deepcopy(coco_gt.dataset['categories']) id = 0 # switch to evaluate mode model.eval() end = time.time() for i, (inputs, anns) in enumerate(val_loader): # forward images one by one (TODO: support batch mode later, or # multiprocess) for j, input in enumerate(inputs): input_anns= anns[j] # anns of this input gt_bbox= np.vstack([ann['bbox'] + [ann['ordered_id']] for ann in input_anns]) im_info= [[input.size(1), input.size(2), input_anns[0]['scale_ratio']]] input_var= Variable(input.unsqueeze(0), requires_grad=False).cuda() cls_prob, bbox_pred, rois = model(input_var, im_info) scores, pred_boxes = model.interpret_outputs(cls_prob, bbox_pred, rois, im_info) print(scores, pred_boxes) # for i in range(scores.shape[0]): # measure elapsed time batch_time.update(time.time() - end) end= time.time() coco_pred.createIndex() coco_eval = COCOeval(coco_gt, coco_pred, 'bbox') coco_eval.params.imgIds= sorted(coco_gt.getImgIds()) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() print('iter: [{0}] ' 'Time {batch_time.avg:.3f} ' 'Val Stats: {1}' .format(i, coco_eval.stats, batch_time=batch_time)) return coco_eval.stats[0]
def _do_eval(res_file, output_dir,_COCO,classes): ## The function is borrowed from https://github.com/rbgirshick/fast-rcnn/ and changed ann_type = 'bbox' coco_dt = _COCO.loadRes(res_file) coco_eval = COCOeval(_COCO, coco_dt) coco_eval.params.useSegm = (ann_type == 'segm') coco_eval.evaluate() coco_eval.accumulate() _print_eval_metrics(coco_eval,classes) # Write the result file eval_file = osp.join(output_dir) eval_result = {} eval_result['precision'] = coco_eval.eval['precision'] eval_result['recall'] = coco_eval.eval['recall'] sio.savemat(eval_file,eval_result) print 'Wrote COCO eval results to: {}'.format(eval_file)
def evaluate_predictions_on_coco( coco_gt, coco_results, json_result_file, iou_type="bbox" ): import json with open(json_result_file, "w") as f: json.dump(coco_results, f) from pycocotools.cocoeval import COCOeval coco_dt = coco_gt.loadRes(str(json_result_file)) # coco_dt = coco_gt.loadRes(coco_results) coco_eval = COCOeval(coco_gt, coco_dt, iou_type) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval
def calc_coco_metrics(coco_annotations, predictions, classes): annotations = ObjectDetectorJson.convert_coco_to_toolbox_format(coco_annotations, classes) detections = [] for annotation, prediction in zip(annotations, predictions): width, height = annotation['image_size'] image_id = annotation['image_id'] for obj_id, obj in enumerate(prediction): label = int(obj[1]) score = float(obj[2]) if obj_id != 0 and score == 0: # At least one prediction must be (COCO API issue) continue bbox = (obj[3:]).tolist() bbox[::2] = [width * i for i in bbox[::2]] bbox[1::2] = [height * i for i in bbox[1::2]] xmin, ymin, xmax, ymax = bbox w_bbox = round(xmax - xmin, 1) h_bbox = round(ymax - ymin, 1) xmin, ymin = round(xmin, 1), round(ymin, 1) coco_det = {} coco_det['image_id'] = image_id coco_det['category_id'] = label coco_det['bbox'] = [xmin, ymin, w_bbox, h_bbox] coco_det['score'] = score detections.append(coco_det) coco_dt = coco_annotations.loadRes(detections) img_ids = sorted(coco_annotations.getImgIds()) coco_eval = COCOeval(coco_annotations, coco_dt, 'bbox') coco_eval.params.imgIds = img_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() metrics = {} for metric_name, value in zip(METRICS_NAMES, coco_eval.stats): metrics[metric_name] = value return metrics
def print_evaluation_scores(json_file): ret = {} assert config.BASEDIR and os.path.isdir(config.BASEDIR) annofile = os.path.join( config.BASEDIR, 'annotations', 'instances_{}.json'.format(config.VAL_DATASET)) coco = COCO(annofile) cocoDt = coco.loadRes(json_file) cocoEval = COCOeval(coco, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() ret['mAP(bbox)'] = cocoEval.stats[0] if config.MODE_MASK: cocoEval = COCOeval(coco, cocoDt, 'segm') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() ret['mAP(segm)'] = cocoEval.stats[0] return ret
def _update(self): """Use coco to get real scores. """ if not self._current_id == len(self._img_ids): warnings.warn( 'Recorded {} out of {} validation images, incompelete results'.format( self._current_id, len(self._img_ids))) import json try: with open(self._filename, 'w') as f: json.dump(self._results, f) except IOError as e: raise RuntimeError("Unable to dump json file, ignored. What(): {}".format(str(e))) pred = self.dataset.coco.loadRes(self._filename) gt = self.dataset.coco # lazy import pycocotools try_import_pycocotools() from pycocotools.cocoeval import COCOeval coco_eval = COCOeval(gt, pred, 'bbox') coco_eval.evaluate() coco_eval.accumulate() self._coco_eval = coco_eval return coco_eval
def evaluate_coco(model, dataset, coco, config, eval_type="bbox", limit=None, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation """ # Pick COCO images from the dataset image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] # Get corresponding COCO image IDs. coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): if i%10==0: print('Processed %d images'%i ) # Load image image = dataset.load_image(image_id) # Run detection t = time.time() r = inference(image, model, config) t_prediction += (time.time() - t) # Convert results to COCO format image_results = build_coco_results(dataset, coco_image_ids[i:i + 1], r["rois"], r["class_ids"], r["scores"], r["masks"]) results.extend(image_results) # Load results. This modifies results with additional attributes. coco_results = coco.loadRes(results) # Evaluate cocoEval = COCOeval(coco, coco_results, eval_type) cocoEval.params.imgIds = coco_image_ids # Only evaluate for person. cocoEval.params.catIds = coco.getCatIds(catNms=['person']) cocoEval.evaluate() a=cocoEval.accumulate() b=cocoEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start)
def print_evaluation_scores(json_file): ret = {} assert config.BASEDIR and os.path.isdir(config.BASEDIR) annofile = os.path.join( config.BASEDIR, 'annotations', 'instances_{}.json'.format(config.VAL_DATASET)) coco = COCO(annofile) cocoDt = coco.loadRes(json_file) cocoEval = COCOeval(coco, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() fields = ['IoU=0.5:0.95', 'IoU=0.5', 'IoU=0.75', 'small', 'medium', 'large'] for k in range(6): ret['mAP(bbox)/' + fields[k]] = cocoEval.stats[k] if config.MODE_MASK: cocoEval = COCOeval(coco, cocoDt, 'segm') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() for k in range(6): ret['mAP(segm)/' + fields[k]] = cocoEval.stats[k] return ret
def main(): # pylint: disable=import-outside-toplevel from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval parser = make_parser() args = parser.parse_args() sys.path.insert(0, os.path.dirname(args.file)) current_network = importlib.import_module( os.path.basename(args.file).split(".")[0]) cfg = current_network.Cfg() if args.weight_file: args.start_epoch = args.end_epoch = -1 else: if args.start_epoch == -1: args.start_epoch = cfg.max_epoch - 1 if args.end_epoch == -1: args.end_epoch = args.start_epoch assert 0 <= args.start_epoch <= args.end_epoch < cfg.max_epoch for epoch_num in range(args.start_epoch, args.end_epoch + 1): if args.weight_file: model_file = args.weight_file else: model_file = "log-of-{}/epoch_{}.pkl".format( os.path.basename(args.file).split(".")[0], epoch_num) logger.info("Load Model : %s completed", model_file) results_list = list() result_queue = Queue(2000) procs = [] for i in range(args.ngpus): proc = Process( target=worker, args=( current_network, model_file, args.dataset_dir, i, args.ngpus, result_queue, ), ) proc.start() procs.append(proc) num_imgs = dict(coco=5000, objects365=30000) for _ in tqdm(range(num_imgs[cfg.test_dataset["name"]])): results_list.append(result_queue.get()) for p in procs: p.join() all_results = DetEvaluator.format(results_list, cfg) json_path = "log-of-{}/epoch_{}.json".format( os.path.basename(args.file).split(".")[0], epoch_num) all_results = json.dumps(all_results) with open(json_path, "w") as fo: fo.write(all_results) logger.info("Save to %s finished, start evaluation!", json_path) eval_gt = COCO( os.path.join(args.dataset_dir, cfg.test_dataset["name"], cfg.test_dataset["ann_file"])) eval_dt = eval_gt.loadRes(json_path) cocoEval = COCOeval(eval_gt, eval_dt, iouType="bbox") cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() metrics = [ "AP", "[email protected]", "[email protected]", "APs", "APm", "APl", "AR@1", "AR@10", "AR@100", "ARs", "ARm", "ARl", ] logger.info("mmAP".center(32, "-")) for i, m in enumerate(metrics): logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i]) logger.info("-" * 32)
def main(): from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval parser = make_parser() args = parser.parse_args() model_name = "{}_{}x{}".format(args.arch, cfg.input_shape[0], cfg.input_shape[1]) save_dir = os.path.join(args.save_dir, model_name) if not os.path.exists(save_dir): os.makedirs(save_dir) mge.set_log_file(os.path.join(save_dir, "log.txt")) args.ngpus = (dist.helper.get_device_count_by_fork("gpu") if args.ngpus is None else args.ngpus) cfg.batch_size = cfg.batch_size if args.batch_size is None else args.batch_size dt_path = os.path.join(cfg.data_root, "person_detection_results", args.dt_file) dets = json.load(open(dt_path, "r")) gt_path = os.path.join(cfg.data_root, "annotations", "person_keypoints_val2017.json") eval_gt = COCO(gt_path) gt = eval_gt.dataset dets = [ i for i in dets if (i["image_id"] in eval_gt.imgs and i["category_id"] == 1) ] ann_file = {"images": gt["images"], "annotations": dets} if args.end_epoch == -1: args.end_epoch = args.start_epoch for epoch_num in range(args.start_epoch, args.end_epoch + 1, args.test_freq): if args.model: model_file = args.model else: model_file = "{}/epoch_{}.pkl".format(args.model_dir, epoch_num) logger.info("Load Model : %s completed", model_file) dist_worker = dist.launcher(n_gpus=args.ngpus)(worker) all_results = dist_worker(args.arch, model_file, cfg.data_root, ann_file) all_results = sum(all_results, []) json_name = "log-of-{}_epoch_{}.json".format(args.arch, epoch_num) json_path = os.path.join(save_dir, json_name) all_results = json.dumps(all_results) with open(json_path, "w") as fo: fo.write(all_results) logger.info("Save to %s finished, start evaluation!", json_path) eval_dt = eval_gt.loadRes(json_path) cocoEval = COCOeval(eval_gt, eval_dt, iouType="keypoints") cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() metrics = [ "AP", "[email protected]", "[email protected]", "APm", "APl", "AR", "[email protected]", "[email protected]", "ARm", "ARl", ] logger.info("mmAP".center(32, "-")) for i, m in enumerate(metrics): logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i]) logger.info("-" * 32)
def run( data, weights=None, # model.pt path(s) batch_size=32, # batch size imgsz=640, # inference size (pixels) conf_thres=0.001, # confidence threshold iou_thres=0.6, # NMS IoU threshold task='val', # train, val, test, speed or study device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu single_cls=False, # treat as single-class dataset augment=False, # augmented inference verbose=False, # verbose output save_txt=False, # save results to *.txt save_hybrid=False, # save label+prediction hybrid results to *.txt save_conf=False, # save confidences in --save-txt labels save_json=False, # save a COCO-JSON results file project=ROOT / 'runs/val', # save to project/name name='exp', # save to project/name exist_ok=False, # existing project/name ok, do not increment half=True, # use FP16 half-precision inference model=None, dataloader=None, save_dir=Path(''), plots=True, callbacks=Callbacks(), compute_loss=None, ): # Initialize/load model and set device training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly device = select_device(device, batch_size=batch_size) # Directories save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir( parents=True, exist_ok=True) # make dir # Load model check_suffix(weights, '.pt') model = attempt_load(weights, map_location=device) # load FP32 model gs = max(int(model.stride.max()), 32) # grid size (max stride) imgsz = check_img_size(imgsz, s=gs) # check image size # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 # if device.type != 'cpu' and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) # Data data = check_dataset(data) # check # Half half &= device.type != 'cpu' # half precision only supported on CUDA model.half() if half else model.float() # Configure model.eval() is_coco = isinstance(data.get('val'), str) and data['val'].endswith( 'coco/val2017.txt') # COCO dataset nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Dataloader if not training: if device.type != 'cpu': model( torch.zeros(1, 3, imgsz, imgsz).to(device).type_as( next(model.parameters()))) # run once pad = 0.0 if task == 'speed' else 0.5 task = task if task in ( 'train', 'val', 'test') else 'val' # path to train/val/test images dataloader = create_dataloader(data[task], imgsz, batch_size, gs, single_cls, pad=pad, rect=True, prefix=colorstr(f'{task}: '))[0] seen = 0 confusion_matrix = ConfusionMatrix(nc=nc) names = { k: v for k, v in enumerate( model.names if hasattr(model, 'names') else model.module.names) } class_map = coco80_to_coco91_class() if is_coco else list(range(1000)) s = ('%20s' + '%11s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', '[email protected]', '[email protected]:.95') dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0, 0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): t1 = time_sync() img = img.to(device, non_blocking=True) img = img.half() if half else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width t2 = time_sync() dt[0] += t2 - t1 # Run model out, train_out = model( img, augment=augment) # inference and training outputs dt[1] += time_sync() - t2 # Compute loss if compute_loss: loss += compute_loss([x.float() for x in train_out], targets)[1] # box, obj, cls # Run NMS targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling t3 = time_sync() out = non_max_suppression(out, conf_thres, iou_thres, labels=lb, multi_label=True, agnostic=single_cls) dt[2] += time_sync() - t3 # Statistics per image for si, pred in enumerate(out): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class path, shape = Path(paths[si]), shapes[si][0] seen += 1 if len(pred) == 0: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Predictions if single_cls: pred[:, 5] = 0 predn = pred.clone() scale_coords(img[si].shape[1:], predn[:, :4], shape, shapes[si][1]) # native-space pred # Evaluate if nl: tbox = xywh2xyxy(labels[:, 1:5]) # target boxes scale_coords(img[si].shape[1:], tbox, shape, shapes[si][1]) # native-space labels labelsn = torch.cat((labels[:, 0:1], tbox), 1) # native-space labels correct = process_batch(predn, labelsn, iouv) if plots: confusion_matrix.process_batch(predn, labelsn) else: correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool) stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # (correct, conf, pcls, tcls) # Save/log if save_txt: save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / (path.stem + '.txt')) if save_json: save_one_json(predn, jdict, path, class_map) # append to COCO-JSON dictionary callbacks.run('on_val_image_end', pred, predn, path, names, img[si]) # Plot images if plots and batch_i < 3: f = save_dir / f'val_batch{batch_i}_labels.jpg' # labels Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start() f = save_dir / f'val_batch{batch_i}_pred.jpg' # predictions Thread(target=plot_images, args=(img, output_to_target(out), paths, f, names), daemon=True).start() # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%11i' * 2 + '%11.3g' * 4 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in dt) # speeds per image if not training: shape = (batch_size, 3, imgsz, imgsz) print( f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}' % t) # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) callbacks.run('on_val_end') # Save JSON if save_json and len(jdict): w = Path(weights[0] if isinstance(weights, list) else weights ).stem if weights is not None else '' # weights anno_json = str( Path(data.get('path', '../coco')) / 'annotations/instances_val2017.json') # annotations json pred_json = str(save_dir / f"{w}_predictions.json") # predictions json print(f'\nEvaluating pycocotools mAP... saving {pred_json}...') with open(pred_json, 'w') as f: json.dump(jdict, f) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb check_requirements(['pycocotools']) from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval anno = COCO(anno_json) # init annotations api pred = anno.loadRes(pred_json) # init predictions api eval = COCOeval(anno, pred, 'bbox') if is_coco: eval.params.imgIds = [ int(Path(x).stem) for x in dataloader.dataset.img_files ] # image IDs to evaluate eval.evaluate() eval.accumulate() eval.summarize() map, map50 = eval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: print(f'pycocotools unable to run: {e}') # Return results model.float() # for training if not training: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' print(f"Results saved to {colorstr('bold', save_dir)}{s}") maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(coco_gt, coco_dt, parameters, catIds=None): LOG = getLogger('processor.EvaluateSegmentation') LOG.info("comparing segmentations") stats = dict(parameters) coco_eval = COCOeval(coco_gt, coco_dt, 'segm') # bbox if catIds: coco_eval.params.catIds = catIds #coco_eval.params.iouThrs = [.5:.05:.95] #coco_eval.params.iouThrs = np.linspace(.3, .95, 14) coco_eval.params.maxDets = [ None ] # unlimited nr of detections (requires pycocotools#559) #coco_eval.params.areaRng = [(0, np.inf)] # unlimited region size #coco_eval.params.areaRngLbl = ['all'] # unlimited region size # Note: The IoU threshold criterion is inadequate for flat segmentation, # because over-/undersegmentation can quickly become false negative/positive. # The pycocotools implementation is especially inadequate, because # it only counts 1:1 matches (and not even the largest or best-scoring, #564). # On the other hand, purely pixel-wise measures do not distinguish instances, # i.e. neighbours can quickly become merged or instances torn apart. # Our approach therefore does not build on pycocotools for matching # and aggregation, only for fast IoU calculation. All non-zero pairs # are considered matches if their intersection over union > 0.5 _or_ # their intersection over either side > 0.5. Matches can thus be n:m. # Non-matches are counted as well (false positives and false negatives). # Aggregation uses microaveraging over images. Besides counting segments, # the pixel areas are counted and averaged (as ratios). # FIXME: We must differentiate between allowable and non-allowable over/under-segmentation (splits/merges). # (A region's split is allowable if it flows in the textLineOrder of the respective GT, # i.e. lines are likely to be either on one side or the other, but not both. # For top-to-bottom/bottom-to-top regions, vertical splits are allowable. # For left-to-right/right-to-left regions, horizontal splits are allowable. # To be sure, we could also validate that explicitly – evaluating both levels at the same time. # Analogously, a number of regions' merge is allowable if it flows in the textLineOrder # of them all, and the GT global reading order has no other regions in between. # For top-to-bottom/bottom-to-top regions, vertical merges are allowable. # For left-to-right/right-to-left regions, horizontal merges are allowable. # Again, we could also validate that the overall textline flow is equivalent.) # This difference can in turn be used to weigh a match pair's score accordingly # when aggregating. For precision-like scores, we would rule out non-allowable merges # (by counting them as FP), and for recall-like scores, we would rule out non-allowable splits # (by counting them as FN). # We can also weigh these non-allowable cases by their share of height # (in vertical textLineOrder and horizontal writing) or width # (in horizontal textLineOrder and vertical writing) which is in disagreement, # or the share of its textlines that have been split or lost. # Furthermore, we can weigh matches by the share of non-text regions or fg pixels involved. coco_eval.evaluate() # get by-page alignment (ignoring inadequate 1:1 matching by pycocotools) def get(arg): return lambda x: x[arg] numImgs = len(coco_eval.params.imgIds) numAreas = len(coco_eval.params.areaRng) for imgind, imgId in enumerate(coco_eval.params.imgIds): img = coco_gt.imgs[imgId] pageId = img['file_name'] for catind, catId in enumerate(coco_eval.params.catIds): cat = coco_gt.cats[catId] catName = cat['name'] if not catId: continue # bypassing COCOeval.evaluateImg, hook onto its results # (again, we stay at areaRng[0]=all and maxDets[0]=all) start = catind * numImgs * numAreas evalimg = coco_eval.evalImgs[start + imgind] if evalimg is None: continue # no DT and GT here # record as dict by pageId / by category imgstats = stats.setdefault('by-image', dict()) pagestats = imgstats.setdefault(pageId, dict()) # get matches and ious and scores ious = coco_eval.ious[imgId, catId] if len(ious): overlaps_dt, overlaps_gt = ious.nonzero() else: overlaps_dt = overlaps_gt = [] # reconstruct score sorting in computeIoU gt = coco_eval._gts[imgId, catId] dt = coco_eval._dts[imgId, catId] dtind = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in dtind] matches = list() gtmatches = dict() dtmatches = dict() for dtind, gtind in zip(overlaps_dt, overlaps_gt): d = dt[dtind] g = gt[gtind] iou = ious[dtind, gtind] union = maskArea( mergeMasks([g['segmentation'], d['segmentation']])) intersection = int(iou * union) # cannot use g or d['area'] here, because mask might be fractional (only-fg) instead of outline areag = int(maskArea(g['segmentation'])) aread = int(maskArea(d['segmentation'])) iogt = intersection / areag iodt = intersection / aread if iou < 0.5 and iogt < 0.5 and iodt < 0.5: continue gtmatches.setdefault(gtind, list()).append(dtind) dtmatches.setdefault(dtind, list()).append(gtind) matches.append( (g['id'], d['id'], iogt, iodt, iou, intersection)) pagestats.setdefault('true_positives', dict()).setdefault(catName, list()).append({ 'GT.ID': g['segment_id'], 'DT.ID': d['segment_id'], 'GT.area': areag, 'DT.area': aread, 'I.area': intersection, 'IoGT': iogt, 'IoDT': iodt, 'IoU': iou }) dtmisses = [] for dtind, d in enumerate(dt): if dtind in dtmatches: continue dtmisses.append((d['id'], maskArea(d['segmentation']))) pagestats.setdefault('false_positives', dict()).setdefault(catName, list()).append({ 'DT.ID': d['segment_id'], 'area': int(d['area']) }) gtmisses = [] for gtind, g in enumerate(gt): if gtind in gtmatches: continue gtmisses.append((g['id'], maskArea(g['segmentation']))) pagestats.setdefault('false_negatives', dict()).setdefault(catName, list()).append({ 'GT.ID': g['segment_id'], 'area': int(g['area']) }) # measure under/oversegmentation for this image and category # (follows Zhang et al 2021: Rethinking Semantic Segmentation Evaluation [arXiv:2101.08418]) over_gt = set(gtind for gtind in gtmatches if len(gtmatches[gtind]) > 1) over_dt = set( chain.from_iterable(gtmatches[gtind] for gtind in gtmatches if len(gtmatches[gtind]) > 1)) under_dt = set(dtind for dtind in dtmatches if len(dtmatches[dtind]) > 1) under_gt = set( chain.from_iterable(dtmatches[dtind] for dtind in dtmatches if len(dtmatches[dtind]) > 1)) over_degree = sum(len(gtmatches[gtind]) - 1 for gtind in gtmatches) under_degree = sum( len(dtmatches[dtind]) - 1 for dtind in dtmatches) if len(dt) and len(gt): oversegmentation = len(over_gt) * len(over_dt) / len(gt) / len( dt) undersegmentation = len(under_gt) * len(under_dt) / len( gt) / len(dt) # Zhang's idea of attenuating the under/oversegmentation ratio with a "penalty" # to account for the degree of further sub-segmentation is misguided IMHO, # because its degree term depends on the total number of segments: # oversegmentation = np.tanh(oversegmentation * over_degree) # undersegmentation = np.tanh(undersegmentation * under_degree) pagestats.setdefault('oversegmentation', dict())[catName] = oversegmentation pagestats.setdefault('undersegmentation', dict())[catName] = undersegmentation pagestats.setdefault( 'precision', dict())[catName] = (len(dt) - len(dtmisses)) / len(dt) pagestats.setdefault( 'recall', dict())[catName] = (len(gt) - len(gtmisses)) / len(gt) tparea = sum(map(get(5), matches)) # sum(inter) fparea = sum(map(get(1), dtmisses)) # sum(area) fnarea = sum(map(get(1), gtmisses)) # sum(area) if tparea or (fparea and fnarea): pagestats.setdefault( 'pixel_precision', dict())[catName] = tparea / (tparea + fparea) pagestats.setdefault( 'pixel_recall', dict())[catName] = tparea / (tparea + fnarea) pagestats.setdefault( 'pixel_iou', dict())[catName] = tparea / (tparea + fparea + fnarea) # aggregate per-img/per-cat IoUs for microaveraging evalimg['matches'] = matches # TP evalimg['dtMisses'] = dtmisses # FP evalimg['gtMisses'] = gtmisses # FN evalimg['dtIdsOver'] = [dt[dtind]['id'] for dtind in over_dt] evalimg['gtIdsOver'] = [gt[gtind]['id'] for gtind in over_gt] evalimg['dtIdsUnder'] = [dt[dtind]['id'] for dtind in under_dt] evalimg['gtIdsUnder'] = [gt[gtind]['id'] for gtind in under_gt] catstats = stats.setdefault('by-category', dict()) # accumulate our over-/undersegmentation and IoU ratios numImgs = len(coco_eval.params.imgIds) numAreas = len(coco_eval.params.areaRng) for catind, catId in enumerate(coco_eval.params.catIds): cat = coco_gt.cats[catId] catstats.setdefault(cat['name'], dict()) start = catind * numImgs * numAreas # again, we stay at areaRng[0]=all and maxDets[0]=all evalimgs = [ coco_eval.evalImgs[start + imgind] for imgind in range(numImgs) ] evalimgs = [img for img in evalimgs if img is not None] assert all(img['category_id'] == catId for img in evalimgs) assert all(img['maxDet'] is None for img in evalimgs) assert all(img['aRng'] == coco_eval.params.areaRng[0] for img in evalimgs) if not len(evalimgs): continue # again, we can ignore gtIgnore here, because we only look at areaRng[0]=all # again, we can ignore dtIgnore here, because we only look at maxDet=None numDTs = sum(len(img['dtIds']) for img in evalimgs) numGTs = sum(len(img['gtIds']) for img in evalimgs) overDTs = sum(len(img['dtIdsOver']) for img in evalimgs) overGTs = sum(len(img['gtIdsOver']) for img in evalimgs) underDTs = sum(len(img['dtIdsUnder']) for img in evalimgs) underGTs = sum(len(img['gtIdsUnder']) for img in evalimgs) numIoUs = sum(len(img['matches']) for img in evalimgs) numFPs = sum(len(img['dtMisses']) for img in evalimgs) numFNs = sum(len(img['gtMisses']) for img in evalimgs) sumIoUs = sum(sum(map(get(4), img['matches'])) for img in evalimgs) # sum(iou) sumIoGTs = sum(sum(map(get(2), img['matches'])) for img in evalimgs) # sum(iogt) sumIoDTs = sum(sum(map(get(3), img['matches'])) for img in evalimgs) # sum(iodt) sumTParea = sum(sum(map(get(5), img['matches'])) for img in evalimgs) # sum(inter) sumFParea = sum(sum(map(get(1), img['dtMisses'])) for img in evalimgs) # sum(area) sumFNarea = sum(sum(map(get(1), img['gtMisses'])) for img in evalimgs) # sum(area) if numDTs and numGTs: oversegmentation = overDTs * overGTs / numDTs / numGTs undersegmentation = underDTs * underGTs / numDTs / numGTs precision = (numDTs - numFPs) / numDTs recall = (numGTs - numFNs) / numGTs else: oversegmentation = undersegmentation = precision = recall = -1 if numIoUs: iou = sumIoUs / numIoUs iogt = sumIoGTs / numIoUs iodt = sumIoDTs / numIoUs else: iou = iogt = iodt = -1 if sumTParea or (sumFParea and sumFNarea): pixel_precision = sumTParea / (sumTParea + sumFParea) pixel_recall = sumTParea / (sumTParea + sumFNarea) pixel_iou = sumTParea / (sumTParea + sumFParea + sumFNarea) else: pixel_precision = pixel_recall = pixel_iou = -1 catstats[cat['name']]['oversegmentation'] = oversegmentation catstats[cat['name']]['undersegmentation'] = undersegmentation catstats[cat['name']]['segment-precision'] = precision catstats[cat['name']]['segment-recall'] = recall catstats[cat['name']]['IoGT'] = iogt # i.e. per-match pixel-recall catstats[cat['name']]['IoDT'] = iodt # i.e. per-match pixel-precision catstats[cat['name']]['IoU'] = iou # i.e. per-match pixel-jaccardindex catstats[cat['name']]['pixel-precision'] = pixel_precision catstats[cat['name']]['pixel-recall'] = pixel_recall catstats[cat['name']]['pixel-iou'] = pixel_iou coco_eval.accumulate() coco_eval.summarize() statInds = np.ones(12, np.bool) statInds[7] = False # AR maxDet[1] statInds[8] = False # AR maxDet[2] coco_eval.stats = coco_eval.stats[statInds] stats['scores'] = dict( zip([ 'Average Precision (AP) @[ IoU=0.50:0.95 | area= all ]', 'Average Precision (AP) @[ IoU=0.50 | area= all ]', 'Average Precision (AP) @[ IoU=0.75 | area= all ]', 'Average Precision (AP) @[ IoU=0.50:0.95 | area= small ]', 'Average Precision (AP) @[ IoU=0.50:0.95 | area=medium ]', 'Average Precision (AP) @[ IoU=0.50:0.95 | area= large ]', 'Average Recall (AR) @[ IoU=0.50:0.95 | area= all ]', 'Average Recall (AR) @[ IoU=0.50:0.95 | area= small ]', 'Average Recall (AR) @[ IoU=0.50:0.95 | area=medium ]', 'Average Recall (AR) @[ IoU=0.50:0.95 | area= large ]', ], coco_eval.stats.tolist())) return stats
item['keypoints'] = res item['image_id'] = image_id item['score'] , visible = compute_oks(res, cocoGt.loadAnns(ann_idx)) if len(visible) != 0: for vis in range(17): item['keypoints'][3* vis + 2] = visible[vis] result.append(item) json.dump(result,fp) fp.close() cocoDt = cocoGt.loadRes(write_json) cocoEval = COCOeval(cocoGt, cocoDt, annType) cocoEval.params.imgIds = keys cocoEval.params.catIds = [1] cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() pred = json.load(open(write_json, 'r')) scores = [ x['score'] for x in pred] ap50 = compute_ap(scores, 0.5) print('ap50 is %f' % ap50) logging.info('ap50 is %f' % ap50) ap = 0 for i in np.arange(0.5,1, 0.05).tolist(): ap = ap + compute_ap(scores, i) ap = ap / len(np.arange(0.5, 1 , 0.05).tolist()) print('ap is %f' % ap) logging.info('ap is %f' % ap)
def infer(model, path, detections_file, resize, max_size, batch_size, mixed_precision=True, is_master=True, world=0, annotations=None, use_dali=True, is_validation=False, verbose=True, rotated_bbox=False): 'Run inference on images from path' backend = 'pytorch' if isinstance(model, Model) or isinstance( model, DDP) else 'tensorrt' stride = model.module.stride if isinstance(model, DDP) else model.stride # Create annotations if none was provided if not annotations: annotations = tempfile.mktemp('.json') images = [{ 'id': i, 'file_name': f } for i, f in enumerate(os.listdir(path))] json.dump({'images': images}, open(annotations, 'w')) # TensorRT only supports fixed input sizes, so override input size accordingly if backend == 'tensorrt': max_size = max(model.input_size) # Prepare dataset if verbose: print('Preparing dataset...') if rotated_bbox: if use_dali: raise NotImplementedError( "This repo does not currently support DALI for rotated bbox.") data_iterator = RotatedDataIterator(path, resize, max_size, batch_size, stride, world, annotations, training=False) else: data_iterator = (DaliDataIterator if use_dali else DataIterator)( path, resize, max_size, batch_size, stride, world, annotations, training=False) if verbose: print(data_iterator) # Prepare model if backend is 'pytorch': # If we are doing validation during training, # no need to register model with AMP again if not is_validation: if torch.cuda.is_available(): model = model.cuda() model = amp.initialize(model, None, opt_level='O2' if mixed_precision else 'O0', keep_batchnorm_fp32=True, verbosity=0) model.eval() if verbose: print(' backend: {}'.format(backend)) print(' device: {} {}'.format( world, 'cpu' if not torch.cuda.is_available() else 'GPU' if world == 1 else 'GPUs')) print(' batch: {}, precision: {}'.format( batch_size, 'unknown' if backend is 'tensorrt' else 'mixed' if mixed_precision else 'full')) print(' BBOX type:', 'rotated' if rotated_bbox else 'axis aligned') print('Running inference...') results = [] profiler = Profiler(['infer', 'fw']) with torch.no_grad(): for i, (data, ids, ratios) in enumerate(data_iterator): # Forward pass profiler.start('fw') scores, boxes, classes = model( data, rotated_bbox) #Need to add model size (B, 3, W, H) profiler.stop('fw') results.append([scores, boxes, classes, ids, ratios]) profiler.bump('infer') if verbose and (profiler.totals['infer'] > 60 or i == len(data_iterator) - 1): size = len(data_iterator.ids) msg = '[{:{len}}/{}]'.format(min((i + 1) * batch_size, size), size, len=len(str(size))) msg += ' {:.3f}s/{}-batch'.format(profiler.means['infer'], batch_size) msg += ' (fw: {:.3f}s)'.format(profiler.means['fw']) msg += ', {:.1f} im/s'.format(batch_size / profiler.means['infer']) print(msg, flush=True) profiler.reset() # Gather results from all devices if verbose: print('Gathering results...') results = [torch.cat(r, dim=0) for r in zip(*results)] if world > 1: for r, result in enumerate(results): all_result = [ torch.ones_like(result, device=result.device) for _ in range(world) ] torch.distributed.all_gather(list(all_result), result) results[r] = torch.cat(all_result, dim=0) if is_master: # Copy buffers back to host results = [r.cpu() for r in results] # Collect detections detections = [] processed_ids = set() for scores, boxes, classes, image_id, ratios in zip(*results): image_id = image_id.item() if image_id in processed_ids: continue processed_ids.add(image_id) keep = (scores > 0).nonzero() scores = scores[keep].view(-1) if rotated_bbox: boxes = boxes[keep, :].view(-1, 6) boxes[:, :4] /= ratios else: boxes = boxes[keep, :].view(-1, 4) / ratios classes = classes[keep].view(-1).int() for score, box, cat in zip(scores, boxes, classes): if rotated_bbox: x1, y1, x2, y2, sin, cos = box.data.tolist() theta = np.arctan2(sin, cos) w = x2 - x1 + 1 h = y2 - y1 + 1 seg = rotate_box([x1, y1, w, h, theta]) else: x1, y1, x2, y2 = box.data.tolist() cat = cat.item() if 'annotations' in data_iterator.coco.dataset: cat = data_iterator.coco.getCatIds()[cat] this_det = { 'image_id': image_id, 'score': score.item(), 'category_id': cat } if rotated_bbox: this_det['bbox'] = [ x1, y1, x2 - x1 + 1, y2 - y1 + 1, theta ] this_det['segmentation'] = [seg] else: this_det['bbox'] = [x1, y1, x2 - x1 + 1, y2 - y1 + 1] detections.append(this_det) if detections: # Save detections if detections_file and verbose: print('Writing {}...'.format(detections_file)) detections = {'annotations': detections} detections['images'] = data_iterator.coco.dataset['images'] if 'categories' in data_iterator.coco.dataset: detections['categories'] = [ data_iterator.coco.dataset['categories'] ] if detections_file: json.dump(detections, open(detections_file, 'w'), indent=4) # Evaluate model on dataset if 'annotations' in data_iterator.coco.dataset: if verbose: print('Evaluating model...') with redirect_stdout(None): coco_pred = data_iterator.coco.loadRes( detections['annotations']) if rotated_bbox: coco_eval = COCOeval(data_iterator.coco, coco_pred, 'segm') else: coco_eval = COCOeval(data_iterator.coco, coco_pred, 'bbox') coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval.stats else: print('No detections!')
def test( data, weights=None, batch_size=32, imgsz=640, conf_thres=0.001, iou_thres=0.6, # for NMS save_json=False, single_cls=False, augment=False, verbose=False, model=None, dataloader=None, save_dir=Path(''), # for saving images save_txt=False, # for auto-labelling save_hybrid=False, # for hybrid auto-labelling save_conf=False, # save auto-label confidences plots=True, wandb_logger=None, compute_loss=None, half_precision=True, is_coco=False): # Initialize/load model and set device training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly set_logging() device = select_device(opt.device, batch_size=batch_size) # Directories save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir( parents=True, exist_ok=True) # make dir # Load model model = attempt_load(weights, map_location=device) # load FP32 model gs = max(int(model.stride.max()), 32) # grid size (max stride) imgsz = check_img_size(imgsz, s=gs) # check img_size # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 # if device.type != 'cpu' and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) # Half half = device.type != 'cpu' and half_precision # half precision only supported on CUDA if half: model.half() # Configure model.eval() if isinstance(data, str): is_coco = data.endswith('coco.yaml') with open(data) as f: data = yaml.load(f, Loader=yaml.SafeLoader) check_dataset(data) # check nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Logging log_imgs = 0 if wandb_logger and wandb_logger.wandb: log_imgs = min(wandb_logger.log_imgs, 100) # Dataloader if not training: if device.type != 'cpu': model( torch.zeros(1, 3, imgsz, imgsz).to(device).type_as( next(model.parameters()))) # run once task = opt.task if opt.task in ( 'train', 'val', 'test') else 'val' # path to train/val/test images dataloader = create_dataloader(data[task], imgsz, batch_size, gs, opt, pad=0.5, rect=True, prefix=colorstr(f'{task}: '))[0] seen = 0 confusion_matrix = ConfusionMatrix(nc=nc) names = { k: v for k, v in enumerate( model.names if hasattr(model, 'names') else model.module.names) } coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', '[email protected]', '[email protected]:.95') p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class, wandb_images = [], [], [], [], [] for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): img = img.to(device, non_blocking=True) img = img.half() if half else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width with torch.no_grad(): # Run model t = time_synchronized() out, train_out = model( img, augment=augment) # inference and training outputs t0 += time_synchronized() - t # Compute loss if compute_loss: loss += compute_loss([x.float() for x in train_out], targets)[1][:3] # box, obj, cls # Run NMS targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels lb = [targets[targets[:, 0] == i, 1:] for i in range(nb) ] if save_hybrid else [] # for autolabelling t = time_synchronized() out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=True) t1 += time_synchronized() - t # Statistics per image for si, pred in enumerate(out): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class path = Path(paths[si]) seen += 1 if len(pred) == 0: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Predictions predn = pred.clone() scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0 ]] # normalization gain whwh for *xyxy, conf, cls in predn.tolist(): xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f: f.write(('%g ' * len(line)).rstrip() % line + '\n') # W&B logging - Media Panel Plots if len( wandb_images ) < log_imgs and wandb_logger.current_epoch > 0: # Check for test operation if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0: box_data = [{ "position": { "minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3] }, "class_id": int(cls), "box_caption": "%s %.3f" % (names[cls], conf), "scores": { "class_score": conf }, "domain": "pixel" } for *xyxy, conf, cls in pred.tolist()] boxes = { "predictions": { "box_data": box_data, "class_labels": names } } # inference-space wandb_images.append( wandb_logger.wandb.Image(img[si], boxes=boxes, caption=path.name)) wandb_logger.log_training_progress( predn, path, names) if wandb_logger and wandb_logger.wandb_run else None # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int( path.stem) if path.stem.isnumeric() else path.stem box = xyxy2xywh(predn[:, :4]) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({ 'image_id': image_id, 'category_id': coco91class[int(p[5])] if is_coco else int(p[5]), 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5) }) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels if plots: confusion_matrix.process_batch( predn, torch.cat((labels[:, 0:1], tbox), 1)) # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view( -1) # prediction indices pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view( -1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(predn[pi, :4], tbox[ti]).max( 1) # best ious, indices # Append detections detected_set = set() for j in (ious > iouv[0]).nonzero(as_tuple=False): d = ti[i[j]] # detected target if d.item() not in detected_set: detected_set.add(d.item()) detected.append(d) correct[ pi[j]] = ious[j] > iouv # iou_thres is 1xn if len( detected ) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append( (correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Plot images if plots and batch_i < 3: f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start() f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions Thread(target=plot_images, args=(img, output_to_target(out), paths, f, names), daemon=True).start() # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%12i' * 2 + '%12.3g' * 4 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple if not training: print( 'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb_logger and wandb_logger.wandb: val_batches = [ wandb_logger.wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg')) ] wandb_logger.log({"Validation": val_batches}) if wandb_images: wandb_logger.log({"Bounding Box Debugger/Images": wandb_images}) # Save JSON if save_json and len(jdict): w = Path(weights[0] if isinstance(weights, list) else weights ).stem if weights is not None else '' # weights anno_json = '../coco/annotations/instances_val2017.json' # annotations json pred_json = str(save_dir / f"{w}_predictions.json") # predictions json print('\nEvaluating pycocotools mAP... saving %s...' % pred_json) with open(pred_json, 'w') as f: json.dump(jdict, f) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval anno = COCO(anno_json) # init annotations api pred = anno.loadRes(pred_json) # init predictions api eval = COCOeval(anno, pred, 'bbox') if is_coco: eval.params.imgIds = [ int(Path(x).stem) for x in dataloader.dataset.img_files ] # image IDs to evaluate eval.evaluate() eval.accumulate() eval.summarize() map, map50 = eval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: print(f'pycocotools unable to run: {e}') # Return results model.float() # for training if not training: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' print(f"Results saved to {save_dir}{s}") maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def main(): args = get_args() cocoGt = COCO( os.path.join(args.coco_dir, "annotations/instances_val2017.json")) if args.use_inv_map: inv_map = [0 ] + cocoGt.getCatIds() # First label in inv_map is not used with open(args.mlperf_accuracy_file, "r") as f: results = json.load(f) detections = [] image_ids = set() seen = set() image_map = cocoGt.dataset["images"] for j in results: idx = j['qsl_idx'] # de-dupe in case loadgen sends the same image multiple times if idx in seen: continue seen.add(idx) # reconstruct from mlperf accuracy log # what is written by the benchmark is an array of float32's: # id, box[0], box[1], box[2], box[3], score, detection_class # note that id is a index into instances_val2017.json, not the actual image_id data = np.frombuffer(bytes.fromhex(j['data']), np.float32) for i in range(0, len(data), 7): image_idx, ymin, xmin, ymax, xmax, score, label = data[i:i + 7] image = image_map[idx] image_id = image["id"] height, width = image["height"], image["width"] ymin *= height xmin *= width ymax *= height xmax *= width loc = os.path.join(args.coco_dir, "val2017", image["file_name"]) label = int(label) if args.use_inv_map: label = inv_map[label] # pycoco wants {imageID,x1,y1,w,h,score,class} detections.append({ "image_id": image_id, "image_loc": loc, "category_id": label, "bbox": [ float(xmin), float(ymin), float(xmax - xmin), float(ymax - ymin) ], "score": float(score) }) image_ids.add(image_id) with open(args.output_file, "w") as fp: json.dump(detections, fp, sort_keys=True, indent=4) cocoDt = cocoGt.loadRes( args.output_file) # Load from file to bypass error with Python3 cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox') cocoEval.params.imgIds = list(image_ids) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("mAP={:.3f}%".format(100. * cocoEval.stats[0])) if args.verbose: print("found and ignored {} dupes".format(len(results) - len(seen)))
def evaluate(self, model): """ COCO average precision (AP) Evaluation. Iterate inference on the test dataset and the results are evaluated by COCO API. Args: model : model object Returns: ap50_95 (float) : calculated COCO AP for IoU=50:95 ap50 (float) : calculated COCO AP for IoU=50 """ model.eval() ids = [] data_dict = [] num_images = len(self.dataset) print('total number of images: %d' % (num_images)) # start testing for index in range(num_images): # all the data in val2017 if index % 500 == 0: print('[Eval: %d / %d]' % (index, num_images)) img, id_ = self.dataset.pull_image(index) # load a batch if self.transform is not None: x = torch.from_numpy( self.transform(img)[0][:, :, (2, 1, 0)]).permute(2, 0, 1) x = x.unsqueeze(0).to(self.device) scale = np.array( [[img.shape[1], img.shape[0], img.shape[1], img.shape[0]]]) id_ = int(id_) ids.append(id_) with torch.no_grad(): outputs = model(x) bboxes, scores, cls_inds = outputs bboxes *= scale for i, box in enumerate(bboxes): x1 = float(box[0]) y1 = float(box[1]) x2 = float(box[2]) y2 = float(box[3]) label = self.dataset.class_ids[int(cls_inds[i])] bbox = [x1, y1, x2 - x1, y2 - y1] score = float(scores[i]) # object score * class score A = { "image_id": id_, "category_id": label, "bbox": bbox, "score": score } # COCO json format data_dict.append(A) annType = ['segm', 'bbox', 'keypoints'] # Evaluate the Dt (detection) json comparing with the ground truth if len(data_dict) > 0: print('evaluating ......') cocoGt = self.dataset.coco # workaround: temporarily write data to json file because pycocotools can't process dict in py36. if self.testset: json.dump(data_dict, open('yolov2_2017.json', 'w')) cocoDt = cocoGt.loadRes('yolov2_2017.json') else: _, tmp = tempfile.mkstemp() json.dump(data_dict, open(tmp, 'w')) cocoDt = cocoGt.loadRes(tmp) cocoEval = COCOeval(self.dataset.coco, cocoDt, annType[1]) cocoEval.params.imgIds = ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() return cocoEval.stats[0], cocoEval.stats[1] else: return 0, 0
def evaluate(self, results, metric='bbox', logger=None, jsonfile_prefix=None, classwise=False, proposal_nums=(100, 300, 1000), iou_thrs=np.arange(0.5, 0.96, 0.05)): """Evaluation in COCO protocol. Args: results (list): Testing results of the dataset. metric (str | list[str]): Metrics to be evaluated. logger (logging.Logger | str | None): Logger used for printing related information during evaluation. Default: None. jsonfile_prefix (str | None): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. classwise (bool): Whether to evaluating the AP for each class. proposal_nums (Sequence[int]): Proposal number used for evaluating recalls, such as recall@100, recall@1000. Default: (100, 300, 1000). iou_thrs (Sequence[float]): IoU threshold used for evaluating recalls. If set to a list, the average recall of all IoUs will also be computed. Default: 0.5. Returns: dict[str: float] """ metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in metrics: if metric not in allowed_metrics: raise KeyError('metric {} is not supported'.format(metric)) result_files, tmp_dir = self.format_results(results, jsonfile_prefix) eval_results = {} cocoGt = self.coco for metric in metrics: msg = 'Evaluating {}...'.format(metric) if logger is None: msg = '\n' + msg print_log(msg, logger=logger) if metric == 'proposal_fast': ar = self.fast_eval_recall(results, proposal_nums, iou_thrs, logger='silent') log_msg = [] for i, num in enumerate(proposal_nums): eval_results['AR@{}'.format(num)] = ar[i] log_msg.append('\nAR@{}\t{:.4f}'.format(num, ar[i])) log_msg = ''.join(log_msg) print_log(log_msg, logger=logger) continue if metric not in result_files: raise KeyError('{} is not in results'.format(metric)) try: cocoDt = cocoGt.loadRes(result_files[metric]) except IndexError: print_log('The testing results of the whole dataset is empty.', logger=logger, level=logging.ERROR) break iou_type = 'bbox' if metric == 'proposal' else metric cocoEval = COCOeval(cocoGt, cocoDt, iou_type) cocoEval.params.imgIds = self.img_ids if metric == 'proposal': cocoEval.params.useCats = 0 cocoEval.params.maxDets = list(proposal_nums) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() metric_items = [ 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ] for i, item in enumerate(metric_items): val = float('{:.3f}'.format(cocoEval.stats[i + 6])) eval_results[item] = val else: # cocoEval.params.catIds = [1] cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() if classwise: # Compute per-category AP pass # TODO metric_items = [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' ] for i in range(len(metric_items)): key = '{}_{}'.format(metric, metric_items[i]) val = float('{:.3f}'.format(cocoEval.stats[i])) eval_results[key] = val eval_results['{}_mAP_copypaste'.format(metric)] = ( '{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' '{ap[4]:.3f} {ap[5]:.3f}').format(ap=cocoEval.stats[:6]) if tmp_dir is not None: tmp_dir.cleanup() return eval_results
class Coco(Base): text_labels_keypoints = [ 'AP', 'AP0.5', 'AP0.75', 'APM', 'APL', 'AR', 'AR0.5', 'AR0.75', 'ARM', 'ARL' ] text_labels_bbox = [ 'AP', 'AP0.5', 'AP0.75', 'APS', 'APM', 'APL', 'ART1', 'ART10', 'AR', 'ARS', 'ARM', 'ARL' ] def __init__(self, coco, *, max_per_image=20, category_ids=None, iou_type='keypoints', small_threshold=0.0, keypoint_oks_sigmas=None): super().__init__() if category_ids is None: category_ids = [1] self.coco = coco self.max_per_image = max_per_image self.category_ids = category_ids self.iou_type = iou_type self.small_threshold = small_threshold self.keypoint_oks_sigmas = keypoint_oks_sigmas self.predictions = [] self.image_ids = [] self.eval = None if self.iou_type == 'keypoints': self.text_labels = self.text_labels_keypoints elif self.iou_type == 'bbox': self.text_labels = self.text_labels_bbox else: LOG.warning('Unknown iou type "%s". Specify text_labels yourself.', self.iou_type) LOG.debug('max = %d, category ids = %s, iou_type = %s', self.max_per_image, self.category_ids, self.iou_type) def _stats(self, predictions=None, image_ids=None): # from pycocotools.cocoeval import COCOeval if predictions is None: predictions = self.predictions if image_ids is None: image_ids = self.image_ids coco_eval = self.coco.loadRes(predictions) self.eval = COCOeval(self.coco, coco_eval, iouType=self.iou_type) LOG.info('cat_ids: %s', self.category_ids) if self.category_ids: self.eval.params.catIds = self.category_ids if self.keypoint_oks_sigmas is not None: self.eval.params.kpt_oks_sigmas = np.asarray( self.keypoint_oks_sigmas) if image_ids is not None: print('image ids', image_ids) self.eval.params.imgIds = image_ids self.eval.evaluate() self.eval.accumulate() self.eval.summarize() return self.eval.stats # pylint: disable=unused-argument def accumulate(self, predictions, image_meta, *, ground_truth=None): image_id = int(image_meta['image_id']) self.image_ids.append(image_id) if self.small_threshold: predictions = [ pred for pred in predictions if pred.scale(v_th=0.01) >= self.small_threshold ] if len(predictions) > self.max_per_image: predictions = predictions[:self.max_per_image] image_annotations = [] for pred in predictions: pred_data = pred.json_data() pred_data['image_id'] = image_id pred_data = { k: v for k, v in pred_data.items() if k in ('category_id', 'score', 'keypoints', 'bbox', 'image_id') } image_annotations.append(pred_data) # force at least one annotation per image (for pycocotools) if not image_annotations: n_keypoints = (len(self.keypoint_oks_sigmas) if self.keypoint_oks_sigmas is not None else 17) image_annotations.append({ 'image_id': image_id, 'category_id': 1, 'keypoints': np.zeros((n_keypoints * 3, )).tolist(), 'bbox': [0, 0, 1, 1], 'score': 0.001, }) if LOG.getEffectiveLevel() == logging.DEBUG: self._stats(image_annotations, [image_id]) LOG.debug(image_meta) self.predictions += image_annotations def write_predictions(self, filename, *, additional_data=None): predictions = [{ k: v for k, v in annotation.items() if k in ('image_id', 'category_id', 'keypoints', 'score') } for annotation in self.predictions] with open(filename + '.pred.json', 'w') as f: json.dump(predictions, f) LOG.info('wrote %s.pred.json', filename) with zipfile.ZipFile(filename + '.zip', 'w') as myzip: myzip.write(filename + '.pred.json', arcname='predictions.json') LOG.info('wrote %s.zip', filename) if additional_data: with open(filename + '.pred_meta.json', 'w') as f: json.dump(additional_data, f) LOG.info('wrote %s.pred_meta.json', filename) def stats(self): data = { 'stats': self._stats().tolist(), 'text_labels': self.text_labels, } return data
class InstanceEvaluator(object): def __init__(self, dataset_json, preds_json): # load dataset ground truths self.dataset = COCO(dataset_json) category_ids = self.dataset.getCatIds() categories = [x['name'] for x in self.dataset.loadCats(category_ids)] self.category_to_id_map = dict(zip(categories, category_ids)) self.classes = ['__background__'] + categories self.num_classes = len(self.classes) # load predictions self.preds = self.dataset.loadRes(preds_json) self.coco_eval = COCOeval(self.dataset, self.preds, 'segm') self.coco_eval.params.maxDets = [1, 50, 255] def evaluate(self): self.coco_eval.evaluate() self.coco_eval.accumulate() def _summarize(self, ap=1, iouThr=None, areaRng='all', maxDets=255): p = self.coco_eval.params iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' titleStr = 'Average Precision' if ap == 1 else 'Average Recall' typeStr = '(AP)' if ap==1 else '(AR)' iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \ if iouThr is None else '{:0.2f}'.format(iouThr) aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] if ap == 1: # dimension of precision: [TxRxKxAxM] s = self.coco_eval.eval['precision'] # IoU if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] s = s[:,:,:,aind,mind] else: # dimension of recall: [TxKxAxM] s = self.coco_eval.eval['recall'] if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] s = s[:,:,aind,mind] if len(s[s>-1])==0: mean_s = -1 else: mean_s = np.mean(s[s>-1]) print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) return mean_s def summarize(self, IoU_lo_thres=0.5, IoU_hi_thres=0.95): def _get_thr_ind(thr): ind = np.where((self.coco_eval.params.iouThrs > thr - 1e-5) & (self.coco_eval.params.iouThrs < thr + 1e-5))[0][0] iou_thr = self.coco_eval.params.iouThrs[ind] assert np.isclose(iou_thr, thr) return ind ind_lo = _get_thr_ind(IoU_lo_thres) ind_hi = _get_thr_ind(IoU_hi_thres) # (iou, recall, cls, area, max_dets) precision = self.coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :, :, 0, 2] ap_mean = np.mean(precision[precision > -1]) print('* MeanAP: {}'.format(ap_mean)) print('* Performance by class:') ap_by_class = [] for cls_ind, cls_name in enumerate(self.classes): if cls_name == '__background__': continue cls_precision = self.coco_eval.eval['precision'][ind_lo: (ind_hi + 1), :, cls_ind - 1, 0, 2] cls_ap = np.mean(cls_precision[cls_precision > -1]) ap_by_class.append(cls_ap) print('{}, AP: {}'.format(cls_name, cls_ap)) ap_by_class = np.asarray(ap_by_class) print('* Performance at different thresholds:') ap_by_thres = np.zeros((12,)) ap_by_thres[0] = self._summarize(1) ap_by_thres[1] = self._summarize(1, iouThr=.5, maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[2] = self._summarize(1, iouThr=.75, maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[3] = self._summarize(1, areaRng='small', maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[4] = self._summarize(1, areaRng='medium', maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[5] = self._summarize(1, areaRng='large', maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[6] = self._summarize(0, maxDets=self.coco_eval.params.maxDets[0]) ap_by_thres[7] = self._summarize(0, maxDets=self.coco_eval.params.maxDets[1]) ap_by_thres[8] = self._summarize(0, maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[9] = self._summarize(0, areaRng='small', maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[10] = self._summarize(0, areaRng='medium', maxDets=self.coco_eval.params.maxDets[2]) ap_by_thres[11] = self._summarize(0, areaRng='large', maxDets=self.coco_eval.params.maxDets[2]) return ap_mean, ap_by_class, ap_by_thres
def do_vg_evaluation( cfg, dataset, predictions, output_folder, logger, iou_types, ): # get zeroshot triplet zeroshot_triplet = torch.load( "maskrcnn_benchmark/data/datasets/evaluation/vg/zeroshot_triplet.pytorch", map_location=torch.device("cpu")).long().numpy() attribute_on = cfg.MODEL.ATTRIBUTE_ON num_attributes = cfg.MODEL.ROI_ATTRIBUTE_HEAD.NUM_ATTRIBUTES # extract evaluation settings from cfg # mode = cfg.TEST.RELATION.EVAL_MODE if cfg.MODEL.ROI_RELATION_HEAD.USE_GT_BOX: if cfg.MODEL.ROI_RELATION_HEAD.USE_GT_OBJECT_LABEL: mode = 'predcls' else: mode = 'sgcls' else: mode = 'sgdet' num_rel_category = cfg.MODEL.ROI_RELATION_HEAD.NUM_CLASSES multiple_preds = cfg.TEST.RELATION.MULTIPLE_PREDS iou_thres = cfg.TEST.RELATION.IOU_THRESHOLD assert mode in {'predcls', 'sgdet', 'sgcls', 'phrdet', 'preddet'} groundtruths = [] for image_id, prediction in enumerate(predictions): img_info = dataset.get_img_info(image_id) image_width = img_info["width"] image_height = img_info["height"] # recover original size which is before transform predictions[image_id] = prediction.resize((image_width, image_height)) gt = dataset.get_groundtruth(image_id, evaluation=True) groundtruths.append(gt) save_output(output_folder, groundtruths, predictions, dataset) result_str = '\n' + '=' * 100 + '\n' if "bbox" in iou_types: # create a Coco-like object that we can use to evaluate detection! anns = [] for image_id, gt in enumerate(groundtruths): labels = gt.get_field('labels').tolist() # integer boxes = gt.bbox.tolist() # xyxy for cls, box in zip(labels, boxes): anns.append({ 'area': (box[3] - box[1] + 1) * (box[2] - box[0] + 1), 'bbox': [box[0], box[1], box[2] - box[0] + 1, box[3] - box[1] + 1], # xywh 'category_id': cls, 'id': len(anns), 'image_id': image_id, 'iscrowd': 0, }) fauxcoco = COCO() fauxcoco.dataset = { 'info': { 'description': 'use coco script for vg detection evaluation' }, 'images': [{ 'id': i } for i in range(len(groundtruths))], 'categories': [{ 'supercategory': 'person', 'id': i, 'name': name } for i, name in enumerate(dataset.ind_to_classes) if name != '__background__'], 'annotations': anns, } fauxcoco.createIndex() # format predictions to coco-like cocolike_predictions = [] for image_id, prediction in enumerate(predictions): box = prediction.convert( 'xywh').bbox.detach().cpu().numpy() # xywh score = prediction.get_field( 'pred_scores').detach().cpu().numpy() # (#objs,) label = prediction.get_field( 'pred_labels').detach().cpu().numpy() # (#objs,) # for predcls, we set label and score to groundtruth if mode == 'predcls': label = prediction.get_field('labels').detach().cpu().numpy() score = np.ones(label.shape[0]) assert len(label) == len(box) image_id = np.asarray([image_id] * len(box)) cocolike_predictions.append( np.column_stack((image_id, box, score, label))) # logger.info(cocolike_predictions) cocolike_predictions = np.concatenate(cocolike_predictions, 0) # evaluate via coco API res = fauxcoco.loadRes(cocolike_predictions) coco_eval = COCOeval(fauxcoco, res, 'bbox') coco_eval.params.imgIds = list(range(len(groundtruths))) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() mAp = coco_eval.stats[1] result_str += 'Detection evaluation mAp=%.4f\n' % mAp result_str += '=' * 100 + '\n' if "relations" in iou_types: result_dict = {} evaluator = {} # tradictional Recall@K eval_recall = SGRecall(result_dict) eval_recall.register_container(mode) evaluator['eval_recall'] = eval_recall # no graphical constraint eval_nog_recall = SGNoGraphConstraintRecall(result_dict) eval_nog_recall.register_container(mode) evaluator['eval_nog_recall'] = eval_nog_recall # test on different distribution eval_zeroshot_recall = SGZeroShotRecall(result_dict) eval_zeroshot_recall.register_container(mode) evaluator['eval_zeroshot_recall'] = eval_zeroshot_recall # test on no graph constraint zero-shot recall eval_ng_zeroshot_recall = SGNGZeroShotRecall(result_dict) eval_ng_zeroshot_recall.register_container(mode) evaluator['eval_ng_zeroshot_recall'] = eval_ng_zeroshot_recall # used by https://github.com/NVIDIA/ContrastiveLosses4VRD for sgcls and predcls eval_pair_accuracy = SGPairAccuracy(result_dict) eval_pair_accuracy.register_container(mode) evaluator['eval_pair_accuracy'] = eval_pair_accuracy # used for meanRecall@K eval_mean_recall = SGMeanRecall(result_dict, num_rel_category, dataset.ind_to_predicates, print_detail=True) eval_mean_recall.register_container(mode) evaluator['eval_mean_recall'] = eval_mean_recall # used for no graph constraint mean Recall@K eval_ng_mean_recall = SGNGMeanRecall(result_dict, num_rel_category, dataset.ind_to_predicates, print_detail=True) eval_ng_mean_recall.register_container(mode) evaluator['eval_ng_mean_recall'] = eval_ng_mean_recall # prepare all inputs global_container = {} global_container['zeroshot_triplet'] = zeroshot_triplet global_container['result_dict'] = result_dict global_container['mode'] = mode global_container['multiple_preds'] = multiple_preds global_container['num_rel_category'] = num_rel_category global_container['iou_thres'] = iou_thres global_container['attribute_on'] = attribute_on global_container['num_attributes'] = num_attributes for groundtruth, prediction in zip(groundtruths, predictions): evaluate_relation_of_one_image(groundtruth, prediction, global_container, evaluator) # calculate mean recall eval_mean_recall.calculate_mean_recall(mode) eval_ng_mean_recall.calculate_mean_recall(mode) # print result result_str += eval_recall.generate_print_string(mode) result_str += eval_nog_recall.generate_print_string(mode) result_str += eval_zeroshot_recall.generate_print_string(mode) result_str += eval_ng_zeroshot_recall.generate_print_string(mode) result_str += eval_mean_recall.generate_print_string(mode) result_str += eval_ng_mean_recall.generate_print_string(mode) if cfg.MODEL.ROI_RELATION_HEAD.USE_GT_BOX: result_str += eval_pair_accuracy.generate_print_string(mode) result_str += '=' * 100 + '\n' logger.info(result_str) if "relations" in iou_types: if output_folder: torch.save(result_dict, os.path.join(output_folder, 'result_dict.pytorch')) return float(np.mean(result_dict[mode + '_recall'][100])) elif "bbox" in iou_types: return float(mAp) else: return -1
def evaluate(self, results, metric='bbox', logger=None, jsonfile_prefix=None, classwise=False, proposal_nums=(100, 300, 1000), iou_thrs=np.arange(0.5, 0.96, 0.05)): """Evaluation in COCO protocol. Args: results (list): Testing results of the dataset. metric (str | list[str]): Metrics to be evaluated. logger (logging.Logger | str | None): Logger used for printing related information during evaluation. Default: None. jsonfile_prefix (str | None): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. classwise (bool): Whether to evaluating the AP for each class. proposal_nums (Sequence[int]): Proposal number used for evaluating recalls, such as recall@100, recall@1000. Default: (100, 300, 1000). iou_thrs (Sequence[float]): IoU threshold used for evaluating recalls. If set to a list, the average recall of all IoUs will also be computed. Default: 0.5. Returns: dict[str: float] """ metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in metrics: if metric not in allowed_metrics: raise KeyError(f'metric {metric} is not supported') result_files, tmp_dir = self.format_results(results, jsonfile_prefix) eval_results = {} cocoGt = self.coco for metric in metrics: msg = f'Evaluating {metric}...' if logger is None: msg = '\n' + msg print_log(msg, logger=logger) if metric == 'proposal_fast': ar = self.fast_eval_recall(results, proposal_nums, iou_thrs, logger='silent') log_msg = [] for i, num in enumerate(proposal_nums): eval_results[f'AR@{num}'] = ar[i] log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') log_msg = ''.join(log_msg) print_log(log_msg, logger=logger) continue if metric not in result_files: raise KeyError(f'{metric} is not in results') try: cocoDt = cocoGt.loadRes(result_files[metric]) except IndexError: print_log('The testing results of the whole dataset is empty.', logger=logger, level=logging.ERROR) break iou_type = 'bbox' if metric == 'proposal' else metric cocoEval = COCOeval(cocoGt, cocoDt, iou_type) cocoEval.params.catIds = self.cat_ids cocoEval.params.imgIds = self.img_ids if metric == 'proposal': cocoEval.params.useCats = 0 cocoEval.params.maxDets = list(proposal_nums) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() metric_items = [ 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ] for i, item in enumerate(metric_items): val = float(f'{cocoEval.stats[i + 6]:.3f}') eval_results[item] = val else: cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() if classwise: # Compute per-category AP # Compute per-category AP # from https://github.com/facebookresearch/detectron2/ precisions = cocoEval.eval['precision'] # precision: (iou, recall, cls, area range, max dets) assert len(self.cat_ids) == precisions.shape[2] results_per_category = [] for idx, catId in enumerate(self.cat_ids): # area range index 0: all area ranges # max dets index -1: typically 100 per image nm = self.coco.loadCats(catId)[0] precision = precisions[:, :, idx, 0, -1] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (f'{nm["name"]}', f'{float(ap):0.3f}')) num_columns = min(6, len(results_per_category) * 2) results_flatten = list( itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest(*[ results_flatten[i::num_columns] for i in range(num_columns) ]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) print_log('\n' + table.table, logger=logger) metric_items = [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' ] for i in range(len(metric_items)): key = f'{metric}_{metric_items[i]}' val = float(f'{cocoEval.stats[i]:.3f}') eval_results[key] = val ap = cocoEval.stats[:6] eval_results[f'{metric}_mAP_copypaste'] = ( f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' f'{ap[4]:.3f} {ap[5]:.3f}') if tmp_dir is not None: tmp_dir.cleanup() return eval_results
def main(args): cocoGt = COCO(args.groundtruth_path) if not args.just_calculate: # coco format gt torch.set_default_tensor_type('torch.FloatTensor') net = network.Network(args) net.cuda('cuda') check_point = torch.load(args.model_path) net.load_state_dict(check_point['state_dict']) os.makedirs(args.output_dir, exist_ok=True) retDets = [] for image_id in tqdm(cocoGt.getImgIds()): img_file_name = cocoGt.loadImgs(ids=[image_id])[0]['file_name'] img_path = join(args.image_dir, img_file_name) output_image_dir = None if args.save_image: output_image_dir = join(args.output_dir, 'predicts') os.makedirs(output_image_dir, exist_ok=True) np.set_printoptions(precision=2, suppress=True) try: detections = inference_image_path( args.image_dir, img_file_name, net, category_id=args.category_id)['dtboxes'] except Exception as e: print('[E]', e) print('[!] load error ', join(args.image_dir, img_file_name)) continue if args.save_image: bboxes, labels = [], [] for res in detections: score = res['score'] if score < args.confidence: continue bbox = res['bbox'] det = { 'image_id': image_id, 'bbox': bbox, 'category_id': args.category_id, 'score': score, } retDets.append(det) if args.save_image: labels.append(args.category_id) bboxes.append([ bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1], score ]) if args.save_image: dst_img_path = join(output_image_dir, basename(img_path)) if not bboxes: bboxes = [[0., 0., 0., 0., 0.]] labels = [args.category_id] bboxes = np.asarray([ xywh2xyxy(det['bbox']) + [det['score']] for det in detections ]) labels = np.asarray([det['category_id'] for det in detections]) # formatting mmcv.imshow_det_bboxes(img_path, bboxes, labels, score_thr=args.confidence, show=False, out_file=dst_img_path) json_dict = { 'images': cocoGt.loadImgs(cocoGt.getImgIds()), 'annotations': retDets, 'categories': [{ 'supercategory': 'person', 'id': args.category_id, }], } output_json_path = join(args.output_dir, 'predictions.json') with open(output_json_path, 'w') as json_fp: json_str = json.dumps(json_dict) json_fp.write(json_str) output_json_path = join(args.output_dir, 'detres.json') with open(output_json_path, 'w') as json_fp: json_str = json.dumps(retDets) json_fp.write(json_str) if args.calculate: cocoDt = cocoGt.loadRes(join(args.output_dir, 'detres.json')) cocoEval = COCOeval(cocoGt, cocoDt, "bbox") cocoEval.params.maxDets = [100, 500, 1000] cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print('Done')
def benchmark_model(frozen_graph, images_dir, annotation_path, batch_size=1, image_shape=None, num_images=4096, tmp_dir='.benchmark_model_tmp_dir', remove_tmp_dir=True, output_path=None): """Computes accuracy and performance statistics Computes accuracy and performance statistics by executing over many images from the MSCOCO dataset defined by images_dir and annotation_path. Args ---- frozen_graph: A GraphDef representing the object detection model to test. Alternatively, a string representing the path to the saved frozen graph. images_dir: A string representing the path of the COCO images directory. annotation_path: A string representing the path of the COCO annotation file. batch_size: An integer representing the batch size to use when feeding images to the model. image_shape: An optional tuple of integers representing a fixed shape to resize all images before testing. num_images: An integer representing the number of images in the dataset to evaluate with. tmp_dir: A string representing the path where the function may create a temporary directory to store intermediate files. output_path: An optional string representing a path to store the statistics in JSON format. Returns ------- statistics: A named dictionary of accuracy and performance statistics computed for the model. """ if os.path.exists(tmp_dir): if not remove_tmp_dir: raise RuntimeError('Temporary directory exists; %s' % tmp_dir) subprocess.call(['rm', '-rf', tmp_dir]) if batch_size > 1 and image_shape is None: raise RuntimeError( 'Fixed image shape must be provided for batch size > 1') from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval coco = COCO(annotation_file=annotation_path) # get list of image ids to use for evaluation image_ids = coco.getImgIds() if num_images > len(image_ids): print( 'Num images provided %d exceeds number in dataset %d, using %d images instead' % (num_images, len(image_ids), len(image_ids))) num_images = len(image_ids) image_ids = image_ids[0:num_images] # load frozen graph from file if string, otherwise must be GraphDef if isinstance(frozen_graph, str): frozen_graph_path = frozen_graph frozen_graph = tf.GraphDef() with open(frozen_graph_path, 'rb') as f: frozen_graph.ParseFromString(f.read()) elif not isinstance(frozen_graph, tf.GraphDef): raise TypeError('Expected frozen_graph to be GraphDef or str') tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True coco_detections = [] # list of all bounding box detections in coco format runtimes = [] # list of runtimes for each batch image_counts = [] # list of number of images in each batch with tf.Graph().as_default() as tf_graph: with tf.Session(config=tf_config) as tf_sess: tf.import_graph_def(frozen_graph, name='') tf_input = tf_graph.get_tensor_by_name(INPUT_NAME + ':0') tf_boxes = tf_graph.get_tensor_by_name(BOXES_NAME + ':0') tf_classes = tf_graph.get_tensor_by_name(CLASSES_NAME + ':0') tf_scores = tf_graph.get_tensor_by_name(SCORES_NAME + ':0') tf_num_detections = tf_graph.get_tensor_by_name( NUM_DETECTIONS_NAME + ':0') # load batches from coco dataset for image_idx in tqdm.tqdm(range(0, len(image_ids), batch_size)): batch_image_ids = image_ids[image_idx:image_idx + batch_size] batch_images = [] batch_coco_images = [] # read images from file for image_id in batch_image_ids: coco_img = coco.imgs[image_id] batch_coco_images.append(coco_img) image_path = os.path.join(images_dir, coco_img['file_name']) image = _read_image(image_path, image_shape) batch_images.append(image) # run once outside of timing to initialize if image_idx == 0: boxes, classes, scores, num_detections = tf_sess.run( [tf_boxes, tf_classes, tf_scores, tf_num_detections], feed_dict={tf_input: batch_images}) # execute model and compute time difference t0 = time.time() boxes, classes, scores, num_detections = tf_sess.run( [tf_boxes, tf_classes, tf_scores, tf_num_detections], feed_dict={tf_input: batch_images}) t1 = time.time() # log runtime and image count runtimes.append(float(t1 - t0)) image_counts.append(len(batch_images)) # add coco detections for this batch to running list batch_coco_detections = [] for i, image_id in enumerate(batch_image_ids): image_width = batch_coco_images[i]['width'] image_height = batch_coco_images[i]['height'] for j in range(int(num_detections[i])): bbox = boxes[i][j] bbox_coco_fmt = [ bbox[1] * image_width, # x0 bbox[0] * image_height, # x1 (bbox[3] - bbox[1]) * image_width, # width (bbox[2] - bbox[0]) * image_height, # height ] coco_detection = { 'image_id': image_id, 'category_id': int(classes[i][j]), 'bbox': bbox_coco_fmt, 'score': float(scores[i][j]) } coco_detections.append(coco_detection) # write coco detections to file subprocess.call(['mkdir', '-p', tmp_dir]) coco_detections_path = os.path.join(tmp_dir, 'coco_detections.json') with open(coco_detections_path, 'w') as f: json.dump(coco_detections, f) # compute coco metrics cocoDt = coco.loadRes(coco_detections_path) eval = COCOeval(coco, cocoDt, 'bbox') eval.params.imgIds = image_ids eval.evaluate() eval.accumulate() eval.summarize() statistics = { 'map': eval.stats[0], 'avg_latency_ms': 1000.0 * np.mean(runtimes), 'avg_throughput_fps': np.sum(image_counts) / np.sum(runtimes) } if output_path is not None: subprocess.call(['mkdir', '-p', os.path.dirname(output_path)]) with open(output_path, 'w') as f: json.dump(statistics, f) subprocess.call(['rm', '-rf', tmp_dir]) return statistics
def test( data, weights=None, batch_size=32, imgsz=640, conf_thres=0.001, iou_thres=0.6, # for NMS save_json=False, single_cls=False, augment=False, verbose=False, model=None, dataloader=None, save_dir=Path(''), # for saving images save_txt=False, # for auto-labelling save_hybrid=False, # for hybrid auto-labelling save_conf=False, # save auto-label confidences plots=True, log_imgs=0): # number of logged images li = [] temp_path = "/content/drive/MyDrive/00Colab Notebooks/07Datasets/COCO/cocomax/" for i in os.listdir(temp_path): li.append(cv2.imread(temp_path + i)) logger.info("log:{}{}".format(len(li), li[0].shape)) # Initialize/load model and set device training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly ###############コメントアウト # set_logging() device = select_device(opt.device, batch_size=batch_size) # Directories save_dir = Path( increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir( parents=True, exist_ok=True) # make dir # Load model model = attempt_load(weights, map_location=device) # load FP32 model imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 # if device.type != 'cpu' and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) # Half half = device.type != 'cpu' # half precision only supported on CUDA if half: model.half() # logger.info("gpu_model:{}".format(gpu_used.gpuinfo())) # Configure model.eval() ###############コメントアウト # is_coco = data.endswith('coco.yaml') # is COCO dataset with open(data) as f: data = yaml.load(f, Loader=yaml.FullLoader) # model dict check_dataset(data) # check nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Logging log_imgs, wandb = min(log_imgs, 100), None # ceil try: import wandb # Weights & Biases except ImportError: log_imgs = 0 # Dataloader if not training: #下二行、デバック img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img _ = model(img.half() if half else img ) if device.type != 'cpu' else None # run once path = data['test'] if opt.task == 'test' else data[ 'val'] # path to val/test images ##################追加 path = "/content/drive/MyDrive/00Colab Notebooks/07Datasets/COCO/cocomax" logger.info(path) time_veri.start_point(time_veri.t_point0) # dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True)[0] dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True, li=li)[0] time_veri.end_point(time_veri.t_point0) seen = 0 confusion_matrix = ConfusionMatrix(nc=nc) names = { k: v for k, v in enumerate( model.names if hasattr(model, 'names') else model.module.names) } # coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', '[email protected]:.95') p, r, f1, mp, mr, map50, map, t0, t1, t_ex1, t_ex2,load_time = 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., [] loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class, wandb_images = [], [], [], [], [] # logger.info("gpu_dataloader:{}".format(gpu_used.gpuinfo())) time_veri.time_start() for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): img = img.to(device, non_blocking=True) img = img.half() if half else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width # gpu_used.gpuinfo() with torch.no_grad(): # Run model inf_out, train_out = model( img, augment=augment) # inference and training outputs ###################################コメントアウト # # Compute loss # if training: # loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # box, obj, cls # Run NMS targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels lb = [targets[targets[:, 0] == i, 1:] for i in range(nb) ] if save_hybrid else [] # for autolabelling output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb) # Statistics per image for si, pred in enumerate(output): ##############################コメントアウト # labels = targets[targets[:, 0] == si, 1:] # nl = len(labels) # tcls = labels[:, 0].tolist() if nl else [] # target class # path = Path(paths[si]) # seen += 1 # if len(pred) == 0: # if nl: # stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) # continue # Predictions→元画像の大きさにリサイズ predn = pred.clone() # scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred ########################引数、元座標うけとり ex_coords = scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred ###############################追加 # continue ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ # # Append to text file # if save_txt: # gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh # for *xyxy, conf, cls in predn.tolist(): # xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh # line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format # with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f: # f.write(('%g ' * len(line)).rstrip() % line + '\n') # # W&B logging # if plots and len(wandb_images) < log_imgs: # box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]}, # "class_id": int(cls), # "box_caption": "%s %.3f" % (names[cls], conf), # "scores": {"class_score": conf}, # "domain": "pixel"} for *xyxy, conf, cls in pred.tolist()] # boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space # wandb_images.append(wandb.Image(img[si], boxes=boxes, caption=path.name)) # # Append to pycocotools JSON dictionary # if save_json: # # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... # image_id = int(path.stem) if path.stem.isnumeric() else path.stem # box = xyxy2xywh(predn[:, :4]) # xywh # box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner # for p, b in zip(pred.tolist(), box.tolist()): # jdict.append({'image_id': image_id, # 'category_id': coco91class[int(p[5])] if is_coco else int(p[5]), # 'bbox': [round(x, 3) for x in b], # 'score': round(p[4], 5)}) # # Assign all predictions as incorrect # correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) # if nl: # detected = [] # target indices # tcls_tensor = labels[:, 0] # # target boxes # tbox = xywh2xyxy(labels[:, 1:5]) # scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels # if plots: # confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1)) # # Per target class # for cls in torch.unique(tcls_tensor): # ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices # pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices # # Search for detections # if pi.shape[0]: # # Prediction to target ious # ious, i = box_iou(predn[pi, :4], tbox[ti]).max(1) # best ious, indices # # Append detections # detected_set = set() # for j in (ious > iouv[0]).nonzero(as_tuple=False): # d = ti[i[j]] # detected target # if d.item() not in detected_set: # detected_set.add(d.item()) # detected.append(d) # correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn # if len(detected) == nl: # all targets already located in image # break # # Append statistics (correct, conf, pcls, tcls) # stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ #####################追加 # time_veri.end_point(time_veri.t_point4) # time_veri2.start_point(time_veri2.t_point0) # continue # # Plot images # if plots and batch_i < 3: # f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels # Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start() # f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions # Thread(target=plot_images, args=(img, output_to_target(output), paths, f, names), daemon=True).start() #####################################追加################################## time_veri.time_end() t_alle = time_synchronized() logger.info("log:{}".format("処理中断終了")) logger.info("一気通貫:{}".format(time_veri.t_end - time_veri.t_start)) temp = time_veri.t_end - time_veri.t_start logger.info("fps22:{}".format(1 / (temp / 500))) # logger.info("gpu_exit:{}".format(gpu_used.gpuinfo())) # logger.info("gpu_max,list:{}{}".format(gpu_used.used_max, gpu_used.used_list)) import sys sys.exit() #####################################追加################################## # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean( 1) # [P, R, [email protected], [email protected]:0.95] mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%12.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple if not training: print( 'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb and wandb.run: wandb.log({"Images": wandb_images}) wandb.log({ "Validation": [ wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg')) ] }) # Save JSON if save_json and len(jdict): w = Path(weights[0] if isinstance(weights, list) else weights ).stem if weights is not None else '' # weights anno_json = '../coco/annotations/instances_val2017.json' # annotations json pred_json = str(save_dir / f"{w}_predictions.json") # predictions json print('\nEvaluating pycocotools mAP... saving %s...' % pred_json) with open(pred_json, 'w') as f: json.dump(jdict, f) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval anno = COCO(anno_json) # init annotations api pred = anno.loadRes(pred_json) # init predictions api eval = COCOeval(anno, pred, 'bbox') if is_coco: eval.params.imgIds = [ int(Path(x).stem) for x in dataloader.dataset.img_files ] # image IDs to evaluate eval.evaluate() eval.accumulate() eval.summarize() map, map50 = eval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: print(f'pycocotools unable to run: {e}') # Return results if not training: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' print(f"Results saved to {save_dir}{s}") model.float() # for training maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def metrics(pred_data): """Calculate mAP of predicted bboxes.""" from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval num_classes = config.num_classes coco_root = config.coco_root data_type = config.val_data_type # Classes need to train or test. val_cls = config.coco_classes val_cls_dict = {} for i, cls in enumerate(val_cls): val_cls_dict[i] = cls anno_json = os.path.join(coco_root, config.instances_set.format(data_type)) coco_gt = COCO(anno_json) classs_dict = {} cat_ids = coco_gt.loadCats(coco_gt.getCatIds()) for cat in cat_ids: classs_dict[cat["name"]] = cat["id"] predictions = [] img_ids = [] for sample in pred_data: pred_boxes = sample['boxes'] box_scores = sample['box_scores'] img_id = sample['img_id'] h, w = sample['image_shape'] pred_boxes = ssd_bboxes_decode(pred_boxes) final_boxes = [] final_label = [] final_score = [] img_ids.append(img_id) for c in range(1, num_classes): class_box_scores = box_scores[:, c] score_mask = class_box_scores > config.min_score class_box_scores = class_box_scores[score_mask] class_boxes = pred_boxes[score_mask] * [h, w, h, w] if score_mask.any(): nms_index = apply_nms(class_boxes, class_box_scores, config.nms_thershold, config.max_boxes) class_boxes = class_boxes[nms_index] class_box_scores = class_box_scores[nms_index] final_boxes += class_boxes.tolist() final_score += class_box_scores.tolist() final_label += [classs_dict[val_cls_dict[c]]] * \ len(class_box_scores) for loc, label, score in zip(final_boxes, final_label, final_score): res = {} res['image_id'] = img_id res['bbox'] = [loc[1], loc[0], loc[3] - loc[1], loc[2] - loc[0]] res['score'] = score res['category_id'] = label predictions.append(res) with open('predictions.json', 'w') as f: json.dump(predictions, f) coco_dt = coco_gt.loadRes('predictions.json') E = COCOeval(coco_gt, coco_dt, iouType='bbox') E.params.imgIds = img_ids E.evaluate() E.accumulate() E.summarize() return E.stats[0]
def evaluate_coco(generator, model, threshold=0.05): """ Use the pycocotools to evaluate a COCO model on a dataset. Args generator : The generator for generating the evaluation data. model : The model to evaluate. threshold : The score threshold to use. """ # start collecting results results = [] image_ids = [] for index in progressbar.progressbar(range(generator.size()), prefix='COCO evaluation: '): image = generator.load_image(index) image = generator.preprocess_image(image) image, scale = generator.resize_image(image) if keras.backend.image_data_format() == 'channels_first': image = image.transpose((2, 0, 1)) # run network boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0)) # correct boxes for image scale boxes /= scale # change to (x, y, w, h) (MS COCO standard) boxes[:, :, 2] -= boxes[:, :, 0] boxes[:, :, 3] -= boxes[:, :, 1] # compute predicted labels and scores for box, score, label in zip(boxes[0], scores[0], labels[0]): # scores are sorted, so we can break if score < threshold: break # append detection for each positively labeled class image_result = { 'image_id' : generator.image_ids[index], 'category_id' : generator.label_to_coco_label(label), 'score' : float(score), 'bbox' : box.tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(generator.image_ids[index]) if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = generator.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval.stats
def evaluate_coco(generator, model, threshold=0.05): # start collecting results results = [] image_ids = [] for index in range(generator.size()): image = generator.load_image(index) image = generator.preprocess_image(image) image, scale = generator.resize_image(image) # run network boxes, scores, labels = model.predict_on_batch( np.expand_dims(image, axis=0)) # correct boxes for image scale boxes /= scale # change to (x, y, w, h) (MS COCO standard) boxes[:, :, 2] -= boxes[:, :, 0] boxes[:, :, 3] -= boxes[:, :, 1] # compute predicted labels and scores for box, score, label in zip(boxes[0], scores[0], labels[0]): # scores are sorted, so we can break if score < threshold: break # append detection for each positively labeled class image_result = { 'image_id': generator.image_ids[index], 'category_id': generator.label_to_coco_label(label), 'score': float(score), 'bbox': box.tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(generator.image_ids[index]) # print progress print('{}/{}'.format(index, generator.size()), end='\r') if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = generator.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format( generator.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval.stats
score_threshold=0.001, iou_threshold=0.45, method='nms') for det in out: x, y, x1, y1, conf, cls = det detection = { "image_id": int(data.images[img_id]), "category_id": int(data._valid_ids[int(cls)]), "bbox": [x, y, x1 - x, y1 - y], "score": float("{:.2f}".format(conf)) } detections.append(detection) json.dump(detections, open('mAP/results.json', 'w')) coco_dets = data.coco.loadRes('mAP/results.json') coco_eval = COCOeval(data.coco, coco_dets, "bbox") coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() ## # Average Precision(AP) @ [IoU = 0.50:0.95 | area = all | maxDets = 100] = 0.343 # Average Precision(AP) @ [IoU = 0.50 | area = all | maxDets = 100] = 0.572 # Average Precision(AP) @ [IoU = 0.75 | area = all | maxDets = 100] = 0.365 # Average Precision(AP) @ [IoU = 0.50:0.95 | area = small | maxDets = 100] = 0.181 # Average Precision(AP) @ [IoU = 0.50:0.95 | area = medium | maxDets = 100] = 0.377 # Average Precision(AP) @ [IoU = 0.50:0.95 | area = large | maxDets = 100] = 0.451 # Average Recall(AR) @ [IoU = 0.50:0.95 | area = all | maxDets = 1] = 0.282 # Average Recall(AR) @ [IoU = 0.50:0.95 | area = all | maxDets = 10] = 0.435 # Average Recall(AR) @ [IoU = 0.50:0.95 | area = all | maxDets = 100] = 0.462 # Average Recall(AR) @ [IoU = 0.50:0.95 | area = small | maxDets = 100] = 0.296 # Average Recall(AR) @ [IoU = 0.50:0.95 | area = medium | maxDets = 100] = 0.490 # Average Recall(AR) @ [IoU = 0.50:0.95 | area = large | maxDets = 100] = 0.573
def test(cfg, data_cfg, weights, batch_size=16, img_size=416, iou_thres=0.5, conf_thres=0.3, nms_thres=0.45, save_json=True, model=None): device = torch_utils.select_device() if model is None: # Initialize model model = Darknet(cfg, img_size).to(device) # Load weights if weights.endswith('.pt'): # pytorch format model.load_state_dict( torch.load(weights, map_location=device)['model']) else: # darknet format _ = load_darknet_weights(model, weights) if torch.cuda.device_count() > 1: model = nn.DataParallel(model) # Configure run data_cfg = parse_data_cfg(data_cfg) nC = int(data_cfg['classes']) # number of classes (80 for COCO) test_path = data_cfg['valid'] # Dataloader dataset = LoadImagesAndLabels(test_path, img_size=img_size) dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=4, pin_memory=False, collate_fn=dataset.collate_fn) model.eval() mean_mAP, mean_R, mean_P, seen = 0.0, 0.0, 0.0, 0 print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP')) mP, mR, mAPs, TP, jdict = [], [], [], [], [] AP_accum, AP_accum_count = np.zeros(nC), np.zeros(nC) coco91class = coco80_to_coco91_class() for imgs, targets, paths, shapes in tqdm(dataloader): t = time.time() targets = targets.to(device) imgs = imgs.to(device) output = model(imgs) output = non_max_suppression(output, conf_thres=conf_thres, nms_thres=nms_thres) # Compute average precision for each sample for si, detections in enumerate(output): labels = targets[targets[:, 0] == si, 1:] seen += 1 if detections is None: # If there are labels but no detections mark as zero AP if len(labels) != 0: mP.append(0), mR.append(0), mAPs.append(0) continue # Get detections sorted by decreasing confidence scores detections = detections[(-detections[:, 4]).argsort()] if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... box = detections[:, :4].clone() # xyxy scale_coords(img_size, box, shapes[si]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner # add to json dictionary for di, d in enumerate(detections): jdict.append({ 'image_id': int(Path(paths[si]).stem.split('_')[-1]), 'category_id': coco91class[int(d[6])], 'bbox': [float3(x) for x in box[di]], 'score': float3(d[4] * d[5]) }) # If no labels add number of detections as incorrect correct = [] if len(labels) == 0: # correct.extend([0 for _ in range(len(detections))]) mP.append(0), mR.append(0), mAPs.append(0) continue else: # Extract target boxes as (x1, y1, x2, y2) target_box = xywh2xyxy(labels[:, 1:5]) * img_size target_cls = labels[:, 0] detected = [] for *pred_box, conf, cls_conf, cls_pred in detections: # Best iou, index between pred and targets iou, bi = bbox_iou(pred_box, target_box).max(0) # If iou > threshold and class is correct mark as correct if iou > iou_thres and cls_pred == target_cls[ bi] and bi not in detected: correct.append(1) detected.append(bi) else: correct.append(0) # Compute Average Precision (AP) per class AP, AP_class, R, P = ap_per_class( tp=np.array(correct), conf=detections[:, 4].cpu().numpy(), pred_cls=detections[:, 6].cpu().numpy(), target_cls=target_cls.cpu().numpy()) # Accumulate AP per class AP_accum_count += np.bincount(AP_class, minlength=nC) AP_accum += np.bincount(AP_class, minlength=nC, weights=AP) # Compute mean AP across all classes in this image, and append to image list mP.append(P.mean()) mR.append(R.mean()) mAPs.append(AP.mean()) # Means of all images mean_P = np.mean(mP) mean_R = np.mean(mR) mean_mAP = np.mean(mAPs) # Print image mAP and running mean mAP print(('%11s%11s' + '%11.3g' * 4 + 's') % (seen, len(dataset), mean_P, mean_R, mean_mAP, time.time() - t)) # Print mAP per class print('\nmAP Per Class:') for i, c in enumerate(load_classes(data_cfg['names'])): if AP_accum_count[i]: print('%15s: %-.4f' % (c, AP_accum[i] / (AP_accum_count[i]))) # Save JSON if save_json: imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataset.img_files] with open('results.json', 'w') as file: json.dump(jdict, file) from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb cocoGt = COCO('../coco/annotations/instances_val2014.json' ) # initialize COCO ground truth api cocoDt = cocoGt.loadRes( 'results.json') # initialize COCO detections api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() # Return mAP return mean_P, mean_R, mean_mAP
def evaluate_coco(generator, model, threshold=0.05): # start collecting results results = [] image_ids = [] for i in range(generator.size()): image = generator.load_image(i) image = generator.preprocess_image(image) image, scale = generator.resize_image(image) # run network _, _, detections = model.predict_on_batch(np.expand_dims(image, axis=0)) # clip to image shape detections[:, :, 0] = np.maximum(0, detections[:, :, 0]) detections[:, :, 1] = np.maximum(0, detections[:, :, 1]) detections[:, :, 2] = np.minimum(image.shape[1], detections[:, :, 2]) detections[:, :, 3] = np.minimum(image.shape[0], detections[:, :, 3]) # correct boxes for image scale detections[0, :, :4] /= scale # change to (x, y, w, h) (MS COCO standard) detections[:, :, 2] -= detections[:, :, 0] detections[:, :, 3] -= detections[:, :, 1] # compute predicted labels and scores for detection in detections[0, ...]: positive_labels = np.where(detection[4:] > threshold)[0] # append detections for each positively labeled class for label in positive_labels: image_result = { 'image_id': generator.image_ids[i], 'category_id': generator.label_to_coco_label(label), 'score': float(detection[4 + label]), 'bbox': (detection[:4]).tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(generator.image_ids[i]) # print progress print('{}/{}'.format(i, generator.size()), end='\r') if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = generator.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format( generator.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize()
def evaluate_coco(generator, model, threshold=0.05): # start collecting results results = [] image_ids = [] for i in range(generator.size()): image = generator.load_image(i) image = generator.preprocess_image(image) image, scale = generator.resize_image(image) # run network _, _, detections = model.predict_on_batch(np.expand_dims(image, axis=0)) # clip to image shape detections[:, :, 0] = np.maximum(0, detections[:, :, 0]) detections[:, :, 1] = np.maximum(0, detections[:, :, 1]) detections[:, :, 2] = np.minimum(image.shape[1], detections[:, :, 2]) detections[:, :, 3] = np.minimum(image.shape[0], detections[:, :, 3]) # correct boxes for image scale detections[0, :, :4] /= scale # change to (x, y, w, h) (MS COCO standard) detections[:, :, 2] -= detections[:, :, 0] detections[:, :, 3] -= detections[:, :, 1] # compute predicted labels and scores for detection in detections[0, ...]: positive_labels = np.where(detection[4:] > threshold)[0] # append detections for each positively labeled class for label in positive_labels: image_result = { 'image_id' : generator.image_ids[i], 'category_id' : generator.label_to_coco_label(label), 'score' : float(detection[4 + label]), 'bbox' : (detection[:4]).tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(generator.image_ids[i]) # print progress print('{}/{}'.format(i, generator.size()), end='\r') if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = generator.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize()
def run( data, weights=None, # model.pt path(s) batch_size=32, # batch size imgsz=640, # inference size (pixels) conf_thres=0.001, # confidence threshold iou_thres=0.6, # NMS IoU threshold task='val', # train, val, test, speed or study device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu workers=8, # max dataloader workers (per RANK in DDP mode) single_cls=False, # treat as single-class dataset augment=False, # augmented inference verbose=False, # verbose output save_txt=False, # save results to *.txt save_hybrid=False, # save label+prediction hybrid results to *.txt save_conf=False, # save confidences in --save-txt labels save_json=False, # save a COCO-JSON results file project=ROOT / 'runs/val', # save to project/name name='exp', # save to project/name exist_ok=False, # existing project/name ok, do not increment half=True, # use FP16 half-precision inference dnn=False, # use OpenCV DNN for ONNX inference model=None, dataloader=None, save_dir=Path(''), plots=True, callbacks=Callbacks(), compute_loss=None, ): # Initialize/load model and set device training = model is not None if training: # called by train.py device, pt, jit, engine = next(model.parameters( )).device, True, False, False # get model device, PyTorch model half &= device.type != 'cpu' # half precision only supported on CUDA model.half() if half else model.float() else: # called directly device = select_device(device, batch_size=batch_size) # Directories save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir( parents=True, exist_ok=True) # make dir # Load model model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) stride, pt, jit, engine = model.stride, model.pt, model.jit, model.engine imgsz = check_img_size(imgsz, s=stride) # check image size half = model.fp16 # FP16 supported on limited backends with CUDA if engine: batch_size = model.batch_size else: device = model.device if not (pt or jit): batch_size = 1 # export.py models default to batch-size 1 LOGGER.info( f'Forcing --batch-size 1 square inference (1,3,{imgsz},{imgsz}) for non-PyTorch models' ) # Data data = check_dataset(data) # check # Configure model.eval() cuda = device.type != 'cpu' is_coco = isinstance(data.get('val'), str) and data['val'].endswith( f'coco{os.sep}val2017.txt') # COCO dataset nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10, device=device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Dataloader if not training: if pt and not single_cls: # check --weights are trained on --data ncm = model.model.nc assert ncm == nc, f'{weights[0]} ({ncm} classes) trained on different --data than what you passed ({nc} ' \ f'classes). Pass correct combination of --weights and --data that are trained together.' model.warmup(imgsz=(1 if pt else batch_size, 3, imgsz, imgsz)) # warmup pad = 0.0 if task in ('speed', 'benchmark') else 0.5 rect = False if task == 'benchmark' else pt # square inference for benchmarks task = task if task in ( 'train', 'val', 'test') else 'val' # path to train/val/test images dataloader = create_dataloader(data[task], imgsz, batch_size, stride, single_cls, pad=pad, rect=rect, workers=workers, prefix=colorstr(f'{task}: '))[0] seen = 0 confusion_matrix = ConfusionMatrix(nc=nc) names = { k: v for k, v in enumerate( model.names if hasattr(model, 'names') else model.module.names) } class_map = coco80_to_coco91_class() if is_coco else list(range(1000)) s = ('%20s' + '%11s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', '[email protected]', '[email protected]:.95') dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0, 0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] callbacks.run('on_val_start') pbar = tqdm(dataloader, desc=s, bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}') # progress bar for batch_i, (im, targets, paths, shapes) in enumerate(pbar): callbacks.run('on_val_batch_start') t1 = time_sync() if cuda: im = im.to(device, non_blocking=True) targets = targets.to(device) im = im.half() if half else im.float() # uint8 to fp16/32 im /= 255 # 0 - 255 to 0.0 - 1.0 nb, _, height, width = im.shape # batch size, channels, height, width t2 = time_sync() dt[0] += t2 - t1 # Inference out, train_out = model(im) if training else model( im, augment=augment, val=True) # inference, loss outputs dt[1] += time_sync() - t2 # Loss if compute_loss: loss += compute_loss([x.float() for x in train_out], targets)[1] # box, obj, cls # NMS targets[:, 2:] *= torch.tensor((width, height, width, height), device=device) # to pixels lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling t3 = time_sync() out = non_max_suppression(out, conf_thres, iou_thres, labels=lb, multi_label=True, agnostic=single_cls) dt[2] += time_sync() - t3 # Metrics for si, pred in enumerate(out): labels = targets[targets[:, 0] == si, 1:] nl, npr = labels.shape[0], pred.shape[ 0] # number of labels, predictions path, shape = Path(paths[si]), shapes[si][0] correct = torch.zeros(npr, niou, dtype=torch.bool, device=device) # init seen += 1 if npr == 0: if nl: stats.append((correct, *torch.zeros( (3, 0), device=device))) continue # Predictions if single_cls: pred[:, 5] = 0 predn = pred.clone() scale_coords(im[si].shape[1:], predn[:, :4], shape, shapes[si][1]) # native-space pred # Evaluate if nl: tbox = xywh2xyxy(labels[:, 1:5]) # target boxes scale_coords(im[si].shape[1:], tbox, shape, shapes[si][1]) # native-space labels labelsn = torch.cat((labels[:, 0:1], tbox), 1) # native-space labels correct = process_batch(predn, labelsn, iouv) if plots: confusion_matrix.process_batch(predn, labelsn) stats.append((correct, pred[:, 4], pred[:, 5], labels[:, 0])) # (correct, conf, pcls, tcls) # Save/log if save_txt: save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / (path.stem + '.txt')) if save_json: save_one_json(predn, jdict, path, class_map) # append to COCO-JSON dictionary callbacks.run('on_val_image_end', pred, predn, path, names, im[si]) # Plot images if plots and batch_i < 3: plot_images(im, targets, paths, save_dir / f'val_batch{batch_i}_labels.jpg', names) # labels plot_images(im, output_to_target(out), paths, save_dir / f'val_batch{batch_i}_pred.jpg', names) # pred callbacks.run('on_val_batch_end') # Compute metrics stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%11i' * 2 + '%11.3g' * 4 # print format LOGGER.info(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): for i, c in enumerate(ap_class): LOGGER.info(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in dt) # speeds per image if not training: shape = (batch_size, 3, imgsz, imgsz) LOGGER.info( f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}' % t) # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) callbacks.run('on_val_end') # Save JSON if save_json and len(jdict): w = Path(weights[0] if isinstance(weights, list) else weights ).stem if weights is not None else '' # weights anno_json = str( Path(data.get('path', '../coco')) / 'annotations/instances_val2017.json') # annotations json pred_json = str(save_dir / f"{w}_predictions.json") # predictions json LOGGER.info(f'\nEvaluating pycocotools mAP... saving {pred_json}...') with open(pred_json, 'w') as f: json.dump(jdict, f) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb check_requirements(['pycocotools']) from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval anno = COCO(anno_json) # init annotations api pred = anno.loadRes(pred_json) # init predictions api eval = COCOeval(anno, pred, 'bbox') if is_coco: eval.params.imgIds = [ int(Path(x).stem) for x in dataloader.dataset.im_files ] # image IDs to evaluate eval.evaluate() eval.accumulate() eval.summarize() map, map50 = eval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: LOGGER.info(f'pycocotools unable to run: {e}') # Return results model.float() # for training if not training: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(dataset, model, threshold=0.05): model.eval() device = [p.device for p in model.parameters()][0] with torch.no_grad(): # start collecting results results = [] image_ids = [] for index in range(len(dataset)): data = dataset[index] scale = data['scale'] # run network scores, labels, boxes = model(data['img'].permute( 2, 0, 1).to(device=device).float().unsqueeze(dim=0)) scores = scores.cpu() labels = labels.cpu() boxes = boxes.cpu() # correct boxes for image scale boxes /= scale if boxes.shape[0] > 0: # change to (x, y, w, h) (MS COCO standard) boxes[:, 2] -= boxes[:, 0] boxes[:, 3] -= boxes[:, 1] # compute predicted labels and scores #for box, score, label in zip(boxes[0], scores[0], labels[0]): for box_id in range(boxes.shape[0]): score = float(scores[box_id]) label = int(labels[box_id]) box = boxes[box_id, :] # scores are sorted, so we can break if score < threshold: break # append detection for each positively labeled class image_result = { 'image_id': dataset.image_ids[index], 'category_id': dataset.label_to_coco_label(label), 'score': float(score), 'bbox': box.tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(dataset.image_ids[index]) # print progress print('{}/{}'.format(index, len(dataset)), end='\r') if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(dataset.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = dataset.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format( dataset.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() model.train() return
def test(cfg, data_cfg, weights=None, batch_size=1, img_size=416, iou_thres=0.4, conf_thres=0.01, nms_thres=0.4, save_json=False, model=None): if model is None: print("model is None,重新创建model") device = torch_utils.select_device() # Initialize model model = Darknet(cfg, img_size).to(device) # Load weights if weights.endswith('.pt'): # pytorch format model.load_state_dict( torch.load(weights, map_location=device)['model']) else: # darknet format _ = load_darknet_weights(model, weights) if torch.cuda.device_count() > 1: model = nn.DataParallel(model) else: device = next(model.parameters()).device # get model device # Configure run data_cfg = parse_data_cfg(data_cfg) nc = int(data_cfg['classes']) # number of classes test_path = data_cfg['test'] # path to test images names = load_classes(data_cfg['names']) # class names # Dataloader print("***---测试图片路径:---***", test_path) dataset = LoadImagesAndLabels(test_path, img_size, batch_size, augment=False, rect=False) dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=12, pin_memory=True, collate_fn=dataset.collate_fn) seen = 0 model.eval() coco91class = coco80_to_coco91_class() print(('%30s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP', 'F1')) loss, p, r, f1, mp, mr, map, mf1 = 0., 0., 0., 0., 0., 0., 0., 0. jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(dataloader, desc='Computing mAP')): targets = targets.to(device) imgs = imgs.to(device) _, _, height, width = imgs.shape # batch size, channels, height, width # Plot images with bounding boxes if batch_i == 0 and not os.path.exists('test_batch0.jpg'): plot_images(imgs=imgs, targets=targets, paths=paths, fname='test_batch0.jpg') # Run model inf_out, train_out = model(imgs) # inference and training outputs # Compute loss if hasattr(model, 'hyp'): # if model has loss hyperparameters cur_loss = compute_loss(train_out, targets, model)[0].item() loss += cur_loss # Run NMS output = non_max_suppression(inf_out, conf_thres=conf_thres, nms_thres=nms_thres) # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append(([], torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file # with open('test.txt', 'a') as file: # [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred] # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(Path(paths[si]).stem.split('_')[-1]) box = pred[:, :4].clone() # xyxy scale_coords(imgs[si].shape[1:], box, shapes[si]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for di, d in enumerate(pred): jdict.append({ 'image_id': image_id, 'category_id': coco91class[int(d[6])], 'bbox': [float3(x) for x in box[di]], 'score': float(d[4]) }) # Assign all predictions as incorrect correct = [0] * len(pred) if nl: detected = [] tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) tbox[:, [0, 2]] *= width tbox[:, [1, 3]] *= height # Search for correct predictions for i, (*pbox, pconf, pcls_conf, pcls) in enumerate(pred): # Break if all targets already located in image if len(detected) == nl: break # Continue if predicted class not among image classes if pcls.item() not in tcls: continue # Best iou, index between pred and targets m = (pcls == tcls_tensor).nonzero().view(-1) iou, bi = bbox_iou(pbox, tbox[m]).max(0) # If iou > threshold and class is correct mark as correct if iou > iou_thres and m[ bi] not in detected: # and pcls == tcls[bi]: correct[i] = 1 detected.append(m[bi]) # Append statistics (correct, conf, pcls, tcls) stats.append((correct, pred[:, 4].cpu(), pred[:, 6].cpu(), tcls)) # Compute statistics stats = [np.concatenate(x, 0) for x in list(zip(*stats))] # to numpy nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() # Print results pf = '%30s' + '%10.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1)) # Print results per class if nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) # Save JSON if save_json and map and len(jdict): imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataset.img_files] with open('results.json', 'w') as file: json.dump(jdict, file) from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb cocoGt = COCO('../coco/annotations/instances_val2014.json' ) # initialize COCO ground truth api cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() map = cocoEval.stats[1] # update mAP to pycocotools mAP # Return results maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map, mf1, loss / len(dataloader)), maps
def eval_mscoco_with_segm(cocoGT, cocoPred): # running evaluation cocoEval = COCOeval(cocoGT, cocoPred, "keypoints") cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def eval(dataset, predictor): gt_coco = { "licenses": { "name": "", "id": 0, "url": "" }, "images": [], "annotations": [], "categories": [] } for i, c in enumerate(dataset.class_names): gt_coco["categories"].append(create_coco_category(i, c)) dt_coco = { "licenses": gt_coco["licenses"], "annotations": [], "categories": gt_coco["categories"] } input_bbox_converter = BboxFormatConvert(source_format=dataset.bbox_format, target_format='coco') output_bbox_converter = BboxFormatConvert(source_format='pascal_voc', target_format='coco') for i in interactive(range(len(dataset))): sample = dataset[i] image = sample['image'] height, width = image.shape[:2] image_record = create_coco_image_record(i, (width, height)) gt_coco["images"].append(image_record) coco_sample = input_bbox_converter(**sample) boxes = coco_sample['bboxes'] labels = coco_sample['category_id'] scores = [1 for _ in labels] gt_anns = create_coco_annotations(i, boxes, labels, scores) gt_coco["annotations"].extend(gt_anns) boxes, labels, probs = predictor.predict(image, prob_threshold=0) boxes = [b.tolist() for b in boxes] labels = labels.tolist() probs = probs.tolist() boxes = output_bbox_converter(image=image, bboxes=boxes)["bboxes"] dt_anns = create_coco_annotations(i, boxes, labels, probs) dt_coco["annotations"].extend(dt_anns) dt_coco.update({"images": gt_coco["images"]}) gt_coco_obj = COCO() gt_coco_obj.dataset = gt_coco gt_coco_obj.createIndex() dt_coco_obj = COCO() dt_coco_obj.dataset = dt_coco dt_coco_obj.createIndex() eval = COCOeval(gt_coco_obj, dt_coco_obj, iouType='bbox') eval.evaluate() eval.accumulate() eval.summarize() result = { "Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ]": eval.stats[0], "Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ]": eval.stats[1], "Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ]": eval.stats[2], "Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ]": eval.stats[3], "Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ]": eval.stats[4], "Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ]": eval.stats[5], "Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ]": eval.stats[6], "Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ]": eval.stats[7], "Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ]": eval.stats[8], "Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ]": eval.stats[9], "Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ]": eval.stats[10], "Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ]": eval.stats[11] } return result
from fast_rcnn.nms_wrapper import nms from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval import numpy as np import skimage.io as io import pylab if __name__ == '__main__': pylab.rcParams['figure.figsize'] = (10.0, 8.0) annType = 'bbox' ground_truth = '/mnt/d/BigData/COCO/instances_train-val2014/annotations/instances_val2014.json' generated_result = '/mnt/c/Users/Lavenger/git/py-faster-rcnn/tools/result.json' cocoGt = COCO(generated_result) cocoDt = cocoGt.loadRes(generated_result) cocoEval = COCOeval(cocoGt,cocoDt) cocoEval.params.imgIds = imgIds cocoEval.params.useSegm = False cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def test( data, weights=None, batch_size=16, imgsz=640, conf_thres=0.001, iou_thres=0.6, # for NMS save_json=False, single_cls=False, augment=False, verbose=False, model=None, dataloader=None, save_dir='', merge=False, save_txt=False): # Initialize/load model and set device training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly set_logging() device = select_device(opt.device, batch_size=batch_size) merge, save_txt = opt.merge, opt.save_txt # use Merge NMS, save *.txt labels if save_txt: out = Path('inference/output') if os.path.exists(out): shutil.rmtree(out) # delete output folder os.makedirs(out) # make new output folder # Remove previous for f in glob.glob(str(Path(save_dir) / 'test_batch*.jpg')): os.remove(f) # Load model model = attempt_load(weights, map_location=device) # load FP32 model imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 # if device.type != 'cpu' and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) # Half half = device.type != 'cpu' # half precision only supported on CUDA if half: model.half() # Configure model.eval() with open(data) as f: data = yaml.load(f, Loader=yaml.FullLoader) # model dict check_dataset(data) # check nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Dataloader if not training: img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img _ = model(img.half() if half else img ) if device.type != 'cpu' else None # run once path = data['test'] if opt.task == 'test' else data[ 'val'] # path to val/test images dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, hyp=None, augment=False, cache=False, pad=0.5, rect=True)[0] seen = 0 names = model.names if hasattr(model, 'names') else model.module.names coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', '[email protected]:.95') p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): img = img.to(device, non_blocking=True) img = img.half() if half else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width whwh = torch.Tensor([width, height, width, height]).to(device) # Disable gradients with torch.no_grad(): # Run model t = time_synchronized() inf_out, train_out = model( img, augment=augment) # inference and training outputs t0 += time_synchronized() - t # Compute loss if training: # if model has loss hyperparameters loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # GIoU, obj, cls # Run NMS t = time_synchronized() output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge) t1 += time_synchronized() - t # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0 ]] # normalization gain whwh x = pred.clone() x[:, :4] = scale_coords(img[si].shape[1:], x[:, :4], shapes[si][0], shapes[si][1]) # to original for *xyxy, conf, cls in x: xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh with open(str(out / Path(paths[si]).stem) + '.txt', 'a') as f: f.write( ('%g ' * 5 + '\n') % (cls, *xywh)) # label format # Clip boxes to image bounds clip_coords(pred, (height, width)) # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = Path(paths[si]).stem box = pred[:, :4].clone() # xyxy scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({ 'image_id': int(image_id) if image_id.isnumeric() else image_id, 'category_id': coco91class[int(p[5])], 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5) }) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view( -1) # prediction indices pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view( -1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(pred[pi, :4], tbox[ti]).max( 1) # best ious, indices # Append detections detected_set = set() for j in (ious > iouv[0]).nonzero(as_tuple=False): d = ti[i[j]] # detected target if d.item() not in detected_set: detected_set.add(d.item()) detected.append(d) correct[ pi[j]] = ious[j] > iouv # iou_thres is 1xn if len( detected ) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append( (correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Plot images if batch_i < 1: f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i) # filename plot_images(img, targets, paths, str(f), names) # ground truth f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i) plot_images(img, output_to_target(output, width, height), paths, str(f), names) # predictions # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): p, r, ap, f1, ap_class = ap_per_class(*stats) p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean( 1) # [P, R, [email protected], [email protected]:0.95] mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%12.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple if not training: print( 'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Save JSON if save_json and len(jdict): f = 'detections_val2017_%s_results.json' % \ (weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '') # filename print('\nCOCO mAP with pycocotools... saving %s...' % f) with open(f, 'w') as file: json.dump(jdict, file) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files] cocoGt = COCO( glob.glob('../coco/annotations/instances_val*.json') [0]) # initialize COCO ground truth api cocoDt = cocoGt.loadRes(f) # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # image IDs to evaluate cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() map, map50 = cocoEval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: print('ERROR: pycocotools unable to run: %s' % e) # Return results model.float() # for training maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def test(cfg, data, weights=None, batch_size=16, img_size=416, conf_thres=0.001, iou_thres=0.6, # for nms save_json=False, single_cls=False, model=None, dataloader=None): # Initialize/load model and set device if model is None: device = torch_utils.select_device(opt.device, batch_size=batch_size) verbose = opt.task == 'test' # Remove previous for f in glob.glob('test_batch*.png'): os.remove(f) # Initialize model model = Darknet(cfg, img_size).to(device) # Load weights attempt_download(weights) if weights.endswith('.pt'): # pytorch format model.load_state_dict(torch.load(weights, map_location=device)['model']) else: # darknet format load_darknet_weights(model, weights) if device.type != 'cpu' and torch.cuda.device_count() > 1: model = nn.DataParallel(model) else: # called by train.py device = next(model.parameters()).device # get model device verbose = False # Configure run data = parse_data_cfg(data) nc = 1 if single_cls else int(data['classes']) # number of classes path = data['valid'] # path to test images names = load_classes(data['names']) # class names iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 iouv = iouv[0].view(1) # comment for [email protected]:0.95 niou = iouv.numel() # Dataloader if dataloader is None: dataset = LoadImagesAndLabels(path, img_size, batch_size, rect=True, single_cls=opt.single_cls) batch_size = min(batch_size, len(dataset)) dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]), pin_memory=True, collate_fn=dataset.collate_fn) seen = 0 model.eval() coco91class = coco80_to_coco91_class() s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'F1') p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): imgs = imgs.to(device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0 targets = targets.to(device) _, _, height, width = imgs.shape # batch size, channels, height, width whwh = torch.Tensor([width, height, width, height]).to(device) # Plot images with bounding boxes f = 'test_batch%g.png' % batch_i # filename if batch_i < 1 and not os.path.exists(f): plot_images(imgs=imgs, targets=targets, paths=paths, fname=f) # Disable gradients with torch.no_grad(): # Run model t = torch_utils.time_synchronized() inf_out, train_out = model(imgs) # inference and training outputs t0 += torch_utils.time_synchronized() - t # Compute loss if hasattr(model, 'hyp'): # if model has loss hyperparameters loss += compute_loss(train_out, targets, model)[1][:3] # GIoU, obj, cls # Run NMS t = torch_utils.time_synchronized() output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres) # nms t1 += torch_utils.time_synchronized() - t # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file # with open('test.txt', 'a') as file: # [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred] # Clip boxes to image bounds clip_coords(pred, (height, width)) # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(Path(paths[si]).stem.split('_')[-1]) box = pred[:, :4].clone() # xyxy scale_coords(imgs[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for di, d in enumerate(pred): jdict.append({'image_id': image_id, 'category_id': coco91class[int(d[5])], 'bbox': [floatn(x, 3) for x in box[di]], 'score': floatn(d[4], 5)}) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero().view(-1) # prediction indices pi = (cls == pred[:, 5]).nonzero().view(-1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious,_,_= box_iou(pred[pi, :4], tbox[ti]) ious, i = ious.max(1) # best ious, indices # Append detections for j in (ious > iouv[0]).nonzero(): d = ti[i[j]] # detected target if d not in detected: detected.append(d) correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn if len(detected) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) if niou > 1: p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]] mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%10.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) # Print speeds if verbose: t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (img_size, img_size, batch_size) # tuple print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Save JSON if save_json and map and len(jdict): print('\nCOCO mAP with pycocotools...') imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataloader.dataset.img_files] with open('results.json', 'w') as file: json.dump(jdict, file) try: from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval except: print('WARNING: missing pycocotools package, can not compute official COCO mAP. See requirements.txt.') # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() mf1, map = cocoEval.stats[:2] # update to pycocotools results ([email protected]:0.95, [email protected]) # Return results maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map, mf1, *(loss.cpu() / len(dataloader)).tolist()), maps
def evaluate_coco(dataset, model, is_cuda, save_folder, threshold=0.05): model.eval() with torch.no_grad(): # start collecting results results = [] image_ids = [] # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in # (x1, y1, x2, y2, score) all_boxes = [[[] for _ in range(len(dataset))] for _ in range(coco['num_classes'])] # timers _t = {'im_detect': Timer(), 'misc': Timer()} # for test, kaidong #pdb.set_trace() # for test, kaidong #for index in range(60): for index in range(len(dataset)): img, gt, h, w = dataset.pull_item(index) #data = dataset[index] x = Variable(img.unsqueeze(0)) if is_cuda: x = x.cuda() _t['im_detect'].tic() detections = model(x).data detect_time = _t['im_detect'].toc(average=False) for j in range(1, detections.size(1)): det = detections[0, j] index_valid = det[:, 0].gt(0.) if index_valid.sum() < 1: continue det = det[index_valid] det[:, 1] *= w det[:, 3] *= w det[:, 2] *= h det[:, 4] *= h # change to coco format, kaidong det[:, 3] -= det[:, 1] det[:, 4] -= det[:, 2] for k in range(det.size(0)): score = det[k, 0] box = det[k, 1:] # append detection for each positively labeled class image_result = { 'image_id': dataset.ids[index], 'category_id': dataset.target_transform.inverse_label_map[j], #'category_id' : COCO_CLASSES[j-1], 'score': float(score), 'bbox': box.tolist(), } # for test, kaidong #if score > 0.5: # print('coco eval', 'res', image_result) # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(dataset.ids[index]) print('im_detect: {:d}/{:d} {:.3f}s'.format( index + 1, len(dataset), detect_time)) if not len(results): return file_json = os.path.join( save_folder, '{}_bbox_results.json'.format(dataset.img_set)) # write output json.dump(results, open(file_json, 'w'), indent=4) # load results in COCO evaluation tool coco_true = dataset.coco coco_pred = coco_true.loadRes(file_json) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() model.train() return
def evaluate(net, dataset, max_num=-1, during_training=False, benchmark=False, cocoapi=False, traditional_nms=False): frame_times = MovingAverage() dataset_size = len(dataset) if max_num < 0 else min(max_num, len(dataset)) dataset_indices = list(range(len(dataset))) dataset_indices = dataset_indices[:dataset_size] progress_bar = ProgressBar(40, dataset_size) if benchmark: timer.disable('Data loading') else: # For each class and iou, stores tuples (score, isPositive) # Index ap_data[type][iouIdx][classIdx] ap_data = { 'box': [[APDataObject() for _ in cfg.dataset.class_names] for _ in iou_thresholds], 'mask': [[APDataObject() for _ in cfg.dataset.class_names] for _ in iou_thresholds] } make_json = Make_json() for i, image_idx in enumerate(dataset_indices): timer.reset() with timer.env('Data loading'): img, gt, gt_masks, h, w, num_crowd = dataset.pull_item(image_idx) batch = Variable(img.unsqueeze(0)) if cuda: batch = batch.cuda() with timer.env('Network forward'): net_outs = net(batch) nms_outs = NMS(net_outs, traditional_nms) if benchmark: prep_benchmark(nms_outs, h, w) else: prep_metrics(ap_data, nms_outs, gt, gt_masks, h, w, num_crowd, dataset.ids[image_idx], make_json, cocoapi) # First couple of images take longer because we're constructing the graph. # Since that's technically initialization, don't include those in the FPS calculations. fps = 0 if i > 1 and not during_training: frame_times.add(timer.total_time()) fps = 1 / frame_times.get_avg() progress = (i + 1) / dataset_size * 100 progress_bar.set_val(i + 1) print('\rProcessing: %s %d / %d (%.2f%%) %.2f fps ' % (repr(progress_bar), i + 1, dataset_size, progress, fps), end='') if benchmark: print('\n\nStats for the last frame:') timer.print_stats() avg_seconds = frame_times.get_avg() print('Average: %5.2f fps, %5.2f ms' % (1 / frame_times.get_avg(), 1000 * avg_seconds)) else: if cocoapi: make_json.dump() print( f'\nJson files dumped, saved in: {json_path}, start evaluting.' ) gt_annotations = COCO(cfg.dataset.valid_info) bbox_dets = gt_annotations.loadRes( f'{json_path}/bbox_detections.json') mask_dets = gt_annotations.loadRes( f'{json_path}/mask_detections.json') print('\nEvaluating BBoxes:') bbox_eval = COCOeval(gt_annotations, bbox_dets, 'bbox') bbox_eval.evaluate() bbox_eval.accumulate() bbox_eval.summarize() print('\nEvaluating Masks:') bbox_eval = COCOeval(gt_annotations, mask_dets, 'segm') bbox_eval.evaluate() bbox_eval.accumulate() bbox_eval.summarize() return table, mask_row = calc_map(ap_data) print(table) return table, mask_row
def main(args): # create model model = network.__dict__[cfg.model](cfg.output_shape, cfg.num_skeleton, pretrained = False) model = torch.nn.DataParallel(model).cuda() test_loader = torch.utils.data.DataLoader( Mscoco(cfg, is_train=False), batch_size=args.batch, shuffle=False, num_workers=args.workers, pin_memory=True) # load training weights checkpoint_file = os.path.join(args.checkpoint, args.test+'.pth.tar') checkpoint = torch.load(checkpoint_file) model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})".format(checkpoint_file, checkpoint['epoch'])) # switch to evaluation mode model.eval() print('testing...') full_result = [] for i, (inputs, meta) in tqdm(enumerate(test_loader)): with torch.no_grad(): input_var = torch.autograd.Variable(inputs.cuda()) # flip the input image if args.flip == True: flip_inputs = inputs.clone() for i, finp in enumerate(flip_inputs): finp = im_to_numpy(finp) finp = cv2.flip(finp, 1) flip_inputs[i] = im_to_torch(finp) flip_input_var = torch.autograd.Variable(flip_inputs.cuda()) # compute output global_outputs, refine_output = model(input_var) score_map = refine_output.data.cpu() score_map = score_map.numpy() if args.flip == True: flip_global_outputs, flip_output = model(flip_input_var) flip_score_map = flip_output.data.cpu() flip_score_map = flip_score_map.numpy() for i, fscore in enumerate(flip_score_map): fscore = fscore.transpose((1,2,0)) fscore = cv2.flip(fscore, 1) fscore = list(fscore.transpose((2,0,1))) for (q, w) in cfg.symmetry: fscore[q], fscore[w] = fscore[w], fscore[q] fscore = np.array(fscore) score_map[i] += fscore score_map[i] /= 2 ids = meta['img_id'].numpy() det_scores = meta['det_scores'] for b in range(inputs.size(0)): details = meta['augmentation_details'] single_result_dict = {} single_result = [] single_map = score_map[b] r0 = single_map.copy() r0 /= 255 r0 += 0.5 v_score = np.zeros(17) for p in range(17): single_map[p] /= np.amax(single_map[p]) border = 10 dr = np.zeros((cfg.output_shape[0] + 2*border, cfg.output_shape[1]+2*border)) dr[border:-border, border:-border] = single_map[p].copy() dr = cv2.GaussianBlur(dr, (21, 21), 0) lb = dr.argmax() y, x = np.unravel_index(lb, dr.shape) dr[y, x] = 0 lb = dr.argmax() py, px = np.unravel_index(lb, dr.shape) y -= border x -= border py -= border + y px -= border + x ln = (px ** 2 + py ** 2) ** 0.5 delta = 0.25 if ln > 1e-3: x += delta * px / ln y += delta * py / ln x = max(0, min(x, cfg.output_shape[1] - 1)) y = max(0, min(y, cfg.output_shape[0] - 1)) resy = float((4 * y + 2) / cfg.data_shape[0] * (details[b][3] - details[b][1]) + details[b][1]) resx = float((4 * x + 2) / cfg.data_shape[1] * (details[b][2] - details[b][0]) + details[b][0]) v_score[p] = float(r0[p, int(round(y)+1e-10), int(round(x)+1e-10)]) single_result.append(resx) single_result.append(resy) single_result.append(1) if len(single_result) != 0: single_result_dict['image_id'] = int(ids[b]) single_result_dict['category_id'] = 1 single_result_dict['keypoints'] = single_result single_result_dict['score'] = float(det_scores[b])*v_score.mean() full_result.append(single_result_dict) result_path = args.result if not isdir(result_path): mkdir_p(result_path) result_file = os.path.join(result_path, 'result.json') with open(result_file,'w') as wf: json.dump(full_result, wf) # evaluate on COCO eval_gt = COCO(cfg.ori_gt_path) eval_dt = eval_gt.loadRes(result_file) cocoEval = COCOeval(eval_gt, eval_dt, iouType='keypoints') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()