def evaluate(): cocoGt = COCO('annotations.json') cocoDt = cocoGt.loadRes('detections.json') cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def coco_evaluate(json_dataset, res_file, image_ids): coco_dt = json_dataset.COCO.loadRes(str(res_file)) coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval
def compute_ap(self): coco_res = self.loader.coco.loadRes(self.filename) cocoEval = COCOeval(self.loader.coco, coco_res) cocoEval.params.imgIds = self.loader.get_filenames() cocoEval.params.useSegm = False cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() return cocoEval
def _do_coco_eval(self, dtFile, output_dir): """ Evaluate using COCO API """ if self._image_set == 'train' or self._image_set == 'val': cocoGt = self._coco[0] cocoDt = COCO(dtFile) E = COCOeval(cocoGt, cocoDt) E.evaluate() E.accumulate() E.summarize()
def evaluate_detections(self, all_boxes, output_dir=None): resFile = self._write_coco_results_file(all_boxes) cocoGt = self._annotations cocoDt = cocoGt.loadRes(resFile) # running evaluation cocoEval = COCOeval(cocoGt,cocoDt) # useSegm should default to 0 #cocoEval.params.useSegm = 0 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def cocoval(detected_json): eval_json = config.eval_json eval_gt = COCO(eval_json) eval_dt = eval_gt.loadRes(detected_json) cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox') # cocoEval.params.imgIds = eval_gt.getImgIds() cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def _do_keypoint_eval(json_dataset, res_file, output_dir): ann_type = 'keypoints' imgIds = json_dataset.COCO.getImgIds() imgIds.sort() coco_dt = json_dataset.COCO.loadRes(res_file) coco_eval = COCOeval(json_dataset.COCO, coco_dt, ann_type) coco_eval.params.imgIds = imgIds coco_eval.evaluate() coco_eval.accumulate() eval_file = os.path.join(output_dir, 'keypoint_results.pkl') robust_pickle_dump(coco_eval, eval_file) logger.info('Wrote json eval results to: {}'.format(eval_file)) coco_eval.summarize()
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation limit: if not 0, it's the number of images to use for evaluation """ # Pick COCO images from the dataset image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] # Get corresponding COCO image IDs. coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): # Load image image = dataset.load_image(image_id) # Run detection t = time.time() r = model.detect([image], verbose=0)[0] t_prediction += (time.time() - t) # Convert results to COCO format # Cast masks to uint8 because COCO tools errors out on bool image_results = build_coco_results(dataset, coco_image_ids[i:i + 1], r["rois"], r["class_ids"], r["scores"], r["masks"].astype(np.uint8)) results.extend(image_results) # Load results. This modifies results with additional attributes. coco_results = coco.loadRes(results) # Evaluate cocoEval = COCOeval(coco, coco_results, eval_type) cocoEval.params.imgIds = coco_image_ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start)
def validate(val_loader, model, i, silence=False): batch_time = AverageMeter() coco_gt = val_loader.dataset.coco coco_pred = COCO() coco_pred.dataset['images'] = [img for img in coco_gt.datasets['images']] coco_pred.dataset['categories'] = copy.deepcopy(coco_gt.dataset['categories']) id = 0 # switch to evaluate mode model.eval() end = time.time() for i, (inputs, anns) in enumerate(val_loader): # forward images one by one (TODO: support batch mode later, or # multiprocess) for j, input in enumerate(inputs): input_anns= anns[j] # anns of this input gt_bbox= np.vstack([ann['bbox'] + [ann['ordered_id']] for ann in input_anns]) im_info= [[input.size(1), input.size(2), input_anns[0]['scale_ratio']]] input_var= Variable(input.unsqueeze(0), requires_grad=False).cuda() cls_prob, bbox_pred, rois = model(input_var, im_info) scores, pred_boxes = model.interpret_outputs(cls_prob, bbox_pred, rois, im_info) print(scores, pred_boxes) # for i in range(scores.shape[0]): # measure elapsed time batch_time.update(time.time() - end) end= time.time() coco_pred.createIndex() coco_eval = COCOeval(coco_gt, coco_pred, 'bbox') coco_eval.params.imgIds= sorted(coco_gt.getImgIds()) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() print('iter: [{0}] ' 'Time {batch_time.avg:.3f} ' 'Val Stats: {1}' .format(i, coco_eval.stats, batch_time=batch_time)) return coco_eval.stats[0]
def evaluate_predictions_on_coco( coco_gt, coco_results, json_result_file, iou_type="bbox" ): import json with open(json_result_file, "w") as f: json.dump(coco_results, f) from pycocotools.cocoeval import COCOeval coco_dt = coco_gt.loadRes(str(json_result_file)) # coco_dt = coco_gt.loadRes(coco_results) coco_eval = COCOeval(coco_gt, coco_dt, iou_type) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval
def calc_coco_metrics(coco_annotations, predictions, classes): annotations = ObjectDetectorJson.convert_coco_to_toolbox_format(coco_annotations, classes) detections = [] for annotation, prediction in zip(annotations, predictions): width, height = annotation['image_size'] image_id = annotation['image_id'] for obj_id, obj in enumerate(prediction): label = int(obj[1]) score = float(obj[2]) if obj_id != 0 and score == 0: # At least one prediction must be (COCO API issue) continue bbox = (obj[3:]).tolist() bbox[::2] = [width * i for i in bbox[::2]] bbox[1::2] = [height * i for i in bbox[1::2]] xmin, ymin, xmax, ymax = bbox w_bbox = round(xmax - xmin, 1) h_bbox = round(ymax - ymin, 1) xmin, ymin = round(xmin, 1), round(ymin, 1) coco_det = {} coco_det['image_id'] = image_id coco_det['category_id'] = label coco_det['bbox'] = [xmin, ymin, w_bbox, h_bbox] coco_det['score'] = score detections.append(coco_det) coco_dt = coco_annotations.loadRes(detections) img_ids = sorted(coco_annotations.getImgIds()) coco_eval = COCOeval(coco_annotations, coco_dt, 'bbox') coco_eval.params.imgIds = img_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() metrics = {} for metric_name, value in zip(METRICS_NAMES, coco_eval.stats): metrics[metric_name] = value return metrics
def print_evaluation_scores(json_file): ret = {} assert config.BASEDIR and os.path.isdir(config.BASEDIR) annofile = os.path.join( config.BASEDIR, 'annotations', 'instances_{}.json'.format(config.VAL_DATASET)) coco = COCO(annofile) cocoDt = coco.loadRes(json_file) cocoEval = COCOeval(coco, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() ret['mAP(bbox)'] = cocoEval.stats[0] if config.MODE_MASK: cocoEval = COCOeval(coco, cocoDt, 'segm') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() ret['mAP(segm)'] = cocoEval.stats[0] return ret
def evaluate_coco(model, dataset, coco, config, eval_type="bbox", limit=None, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation """ # Pick COCO images from the dataset image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] # Get corresponding COCO image IDs. coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): if i%10==0: print('Processed %d images'%i ) # Load image image = dataset.load_image(image_id) # Run detection t = time.time() r = inference(image, model, config) t_prediction += (time.time() - t) # Convert results to COCO format image_results = build_coco_results(dataset, coco_image_ids[i:i + 1], r["rois"], r["class_ids"], r["scores"], r["masks"]) results.extend(image_results) # Load results. This modifies results with additional attributes. coco_results = coco.loadRes(results) # Evaluate cocoEval = COCOeval(coco, coco_results, eval_type) cocoEval.params.imgIds = coco_image_ids # Only evaluate for person. cocoEval.params.catIds = coco.getCatIds(catNms=['person']) cocoEval.evaluate() a=cocoEval.accumulate() b=cocoEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start)
def print_evaluation_scores(json_file): ret = {} assert config.BASEDIR and os.path.isdir(config.BASEDIR) annofile = os.path.join( config.BASEDIR, 'annotations', 'instances_{}.json'.format(config.VAL_DATASET)) coco = COCO(annofile) cocoDt = coco.loadRes(json_file) cocoEval = COCOeval(coco, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() fields = ['IoU=0.5:0.95', 'IoU=0.5', 'IoU=0.75', 'small', 'medium', 'large'] for k in range(6): ret['mAP(bbox)/' + fields[k]] = cocoEval.stats[k] if config.MODE_MASK: cocoEval = COCOeval(coco, cocoDt, 'segm') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() for k in range(6): ret['mAP(segm)/' + fields[k]] = cocoEval.stats[k] return ret
def main(): parser = argparse.ArgumentParser() parser.add_argument('--output', default='output', type=str) parser.add_argument('--data', default='val2017', type=str) parser.add_argument('--annotations', default='annotations', type=str) parser.add_argument('--inres', default='512,512', type=str) parser.add_argument('--no-full-resolution', action='store_true') args, _ = parser.parse_known_args() args.inres = tuple(int(x) for x in args.inres.split(',')) if not args.no_full_resolution: args.inres = (None, None) os.makedirs(args.output, exist_ok=True) kwargs = { 'num_stacks': 2, 'cnv_dim': 256, 'weights': 'ctdet_coco', 'inres': args.inres, } heads = {'hm': 80, 'reg': 2, 'wh': 2} out_fn_box = os.path.join( args.output, args.data + '_bbox_results_%s_%s.json' % (args.inres[0], args.inres[1])) model = HourglassNetwork(heads=heads, **kwargs) model = CtDetDecode(model) if args.no_full_resolution: letterbox_transformer = LetterboxTransformer(args.inres[0], args.inres[1]) else: letterbox_transformer = LetterboxTransformer(mode='testing', max_stride=128) fns = sorted(glob(os.path.join(args.data, '*.jpg'))) results = [] for fn in tqdm(fns): img = cv2.imread(fn) image_id = int(os.path.splitext(os.path.basename(fn))[0]) pimg = letterbox_transformer(img) pimg = normalize_image(pimg) pimg = np.expand_dims(pimg, 0) detections = model.predict(pimg)[0] for d in detections: x1, y1, x2, y2, score, cl = d # if score < 0.001: # break x1, y1, x2, y2 = letterbox_transformer.correct_box(x1, y1, x2, y2) cl = int(cl) x1, y1, x2, y2 = float(x1), float(y1), float(x2), float(y2) image_result = { 'image_id': image_id, 'category_id': COCO_IDS[cl + 1], 'score': float(score), 'bbox': [x1, y1, (x2 - x1), (y2 - y1)], } results.append(image_result) if not len(results): print("No predictions were generated.") return # write output with open(out_fn_box, 'w') as f: json.dump(results, f, indent=2) print("Predictions saved to: %s" % out_fn_box) # load results in COCO evaluation tool gt_fn = os.path.join(args.annotations, 'instances_%s.json' % args.data) print("Loading GT: %s" % gt_fn) coco_true = COCO(gt_fn) coco_pred = coco_true.loadRes(out_fn_box) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval.stats
def evaluate(config): is_training = False # Load and initialize network net = ModelMain(config, is_training=is_training) net.train(is_training) # Set data parallel net = nn.DataParallel(net) net = net.cuda() # Restore pretrain model if config["pretrain_snapshot"]: logging.info("Load checkpoint: {}".format(config["pretrain_snapshot"])) state_dict = torch.load(config["pretrain_snapshot"]) net.load_state_dict(state_dict) else: logging.warning("missing pretrain_snapshot!!!") # YOLO loss with 3 scales yolo_losses = [] for i in range(3): yolo_losses.append( YOLOLoss(config["yolo"]["anchors"][i], config["yolo"]["classes"], (config["img_w"], config["img_h"]))) # DataLoader. dataloader = torch.utils.data.DataLoader(COCODataset( config["val_path"], (config["img_w"], config["img_h"]), is_training=False), batch_size=config["batch_size"], shuffle=False, num_workers=8, pin_memory=False) # Coco Prepare. index2category = json.load(open("coco_index2category.json")) # Start the eval loop logging.info("Start eval.") coco_results = [] coco_img_ids = set([]) for step, samples in enumerate(dataloader): images, labels = samples["image"], samples["label"] image_paths, origin_sizes = samples["image_path"], samples[ "origin_size"] with torch.no_grad(): outputs = net(images) output_list = [] for i in range(3): output_list.append(yolo_losses[i](outputs[i])) output = torch.cat(output_list, 1) batch_detections = non_max_suppression(output, config["yolo"]["classes"], conf_thres=0.001, nms_thres=0.45) for idx, detections in enumerate(batch_detections): image_id = int(os.path.basename(image_paths[idx])[-16:-4]) coco_img_ids.add(image_id) if detections is not None: origin_size = eval(origin_sizes[idx]) detections = detections.cpu().numpy() for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections: x1 = x1 / config["img_w"] * origin_size[0] x2 = x2 / config["img_w"] * origin_size[0] y1 = y1 / config["img_h"] * origin_size[1] y2 = y2 / config["img_h"] * origin_size[1] w = x2 - x1 h = y2 - y1 coco_results.append({ "image_id": image_id, "category_id": index2category[str(int(cls_pred.item()))], "bbox": (float(x1), float(y1), float(w), float(h)), "score": float(conf), }) logging.info("Now {}/{}".format(step, len(dataloader))) save_results_path = "coco_results.json" with open(save_results_path, "w") as f: json.dump(coco_results, f, sort_keys=True, indent=4, separators=(',', ':')) logging.info("Save coco format results to {}".format(save_results_path)) # COCO api logging.info("Using coco-evaluate tools to evaluate.") cocoGt = COCO(config["annotation_path"]) cocoDt = cocoGt.loadRes(save_results_path) cocoEval = COCOeval(cocoGt, cocoDt, "bbox") cocoEval.params.imgIds = list(coco_img_ids) # real imgIds cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def evaluate_coco(generator, model, threshold=0.05): # start collecting results results = [] image_ids = [] for i in range(generator.size()): image = generator.load_image(i) image = generator.preprocess_image(image) image, scale = generator.resize_image(image) # run network _, _, detections = model.predict_on_batch(np.expand_dims(image, axis=0)) # clip to image shape detections[:, :, 0] = np.maximum(0, detections[:, :, 0]) detections[:, :, 1] = np.maximum(0, detections[:, :, 1]) detections[:, :, 2] = np.minimum(image.shape[1], detections[:, :, 2]) detections[:, :, 3] = np.minimum(image.shape[0], detections[:, :, 3]) # correct boxes for image scale detections[0, :, :4] /= scale # change to (x, y, w, h) (MS COCO standard) detections[:, :, 2] -= detections[:, :, 0] detections[:, :, 3] -= detections[:, :, 1] # compute predicted labels and scores for detection in detections[0, ...]: positive_labels = np.where(detection[4:] > threshold)[0] # append detections for each positively labeled class for label in positive_labels: image_result = { 'image_id' : generator.image_ids[i], 'category_id' : generator.label_to_coco_label(label), 'score' : float(detection[4 + label]), 'bbox' : (detection[:4]).tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(generator.image_ids[i]) # print progress print('{}/{}'.format(i, generator.size()), end='\r') if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = generator.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize()
def test(cfg, data, weights=None, batch_size=16, imgsz=416, conf_thres=0.001, iou_thres=0.6, # for nms save_json=False, single_cls=False, augment=False, model=None, dataloader=None, multi_label=True): # Initialize/load model and set device if model is None: device = torch_utils.select_device(opt.device, batch_size=batch_size) verbose = opt.task == 'test' # Remove previous for f in glob.glob('test_batch*.jpg'): os.remove(f) # Initialize model model = Darknet(cfg, imgsz) # Load weights attempt_download(weights) if weights.endswith('.pt'): # pytorch format model.load_state_dict(torch.load(weights, map_location=device)['model']) else: # darknet format load_darknet_weights(model, weights) # Fuse model.fuse() model.to(device) if device.type != 'cpu' and torch.cuda.device_count() > 1: model = nn.DataParallel(model) else: # called by train.py device = next(model.parameters()).device # get model device verbose = False # Configure run data = parse_data_cfg(data) nc = 1 if single_cls else int(data['classes']) # number of classes path = data['valid'] # path to test images names = load_classes(data['names']) # class names iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 iouv = iouv[0].view(1) # comment for [email protected]:0.95 niou = iouv.numel() # Dataloader if dataloader is None: dataset = LoadImagesAndLabels(path, imgsz, batch_size, rect=True, single_cls=opt.single_cls) batch_size = min(batch_size, len(dataset)) dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]), pin_memory=True, collate_fn=dataset.collate_fn) seen = 0 model.eval() _ = model(torch.zeros((1, 3, imgsz, imgsz), device=device)) if device.type != 'cpu' else None # run once coco91class = coco80_to_coco91_class() s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'F1') p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s, ncols=150)): imgs = imgs.to(device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = imgs.shape # batch size, channels, height, width whwh = torch.Tensor([width, height, width, height]).to(device) # Disable gradients with torch.no_grad(): # Run model t = torch_utils.time_synchronized() inf_out, train_out = model(imgs, augment=augment) # inference and training outputs t0 += torch_utils.time_synchronized() - t # Compute loss if hasattr(model, 'hyp'): # if model has loss hyperparameters loss += compute_loss(train_out, targets, model)[1][:3] # GIoU, obj, cls # Run NMS t = torch_utils.time_synchronized() output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, multi_label=multi_label) t1 += torch_utils.time_synchronized() - t # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file # with open('test.txt', 'a') as file: # [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred] # Clip boxes to image bounds clip_coords(pred, (height, width)) # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(Path(paths[si]).stem.split('_')[-1]) box = pred[:, :4].clone() # xyxy scale_coords(imgs[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({'image_id': image_id, 'category_id': coco91class[int(p[5])], 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5)}) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero().view(-1) # prediction indices pi = (cls == pred[:, 5]).nonzero().view(-1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1) # best ious, indices # Append detections for j in (ious > iouv[0]).nonzero(): d = ti[i[j]] # detected target if d not in detected: detected.append(d) correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn if len(detected) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Plot images if batch_i < 1: f = 'test_batch%g_gt.jpg' % batch_i # filename plot_images(imgs, targets, paths=paths, names=names, fname=f) # ground truth f = 'test_batch%g_pred.jpg' % batch_i plot_images(imgs, output_to_target(output, width, height), paths=paths, names=names, fname=f) # predictions # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) if niou > 1: p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]] mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%10.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) # Print speeds if verbose or save_json: t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Save JSON if save_json and map and len(jdict): print('\nCOCO mAP with pycocotools...') imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataloader.dataset.img_files] with open('results.json', 'w') as file: json.dump(jdict, file) try: from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() # mf1, map = cocoEval.stats[:2] # update to pycocotools results ([email protected]:0.95, [email protected]) except: print('WARNING: pycocotools must be installed with numpy==1.17 to run correctly. ' 'See https://github.com/cocodataset/cocoapi/issues/356') # Return results maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map, mf1, *(loss.cpu() / len(dataloader)).tolist()), maps
def doEval(self, json_file): cocoDt = self.coco.loadRes(json_file) cocoEval = COCOeval(self.coco, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def test(cfg, data, weights=None, batch_size=16, img_size=416, iou_thres=0.5, conf_thres=0.001, nms_thres=0.5, save_json=False, model=None): # Initialize/load model and set device if model is None: device = torch_utils.select_device() verbose = True # Initialize model model = Darknet(cfg, img_size).to(device) # Load weights if weights.endswith('.pt'): # pytorch format model.load_state_dict( torch.load(weights, map_location=device)['model']) else: # darknet format _ = load_darknet_weights(model, weights) if torch.cuda.device_count() > 1: model = nn.DataParallel(model) else: device = next(model.parameters()).device # get model device verbose = False # Configure run data = parse_data_cfg(data) nc = int(data['classes']) # number of classes test_path = data['valid'] # path to test images names = load_classes(data['names']) # class names # Dataloader dataset = LoadImagesAndLabels(test_path, img_size, batch_size) dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=min(os.cpu_count(), batch_size), pin_memory=True, collate_fn=dataset.collate_fn) seen = 0 model.eval() coco91class = coco80_to_coco91_class() s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP', 'F1') p, r, f1, mp, mr, map, mf1 = 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3) jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): targets = targets.to(device) imgs = imgs.to(device) _, _, height, width = imgs.shape # batch size, channels, height, width # Plot images with bounding boxes if batch_i == 0 and not os.path.exists('test_batch0.jpg'): plot_images(imgs=imgs, targets=targets, paths=paths, fname='test_batch0.jpg') # Run model inf_out, train_out = model(imgs) # inference and training outputs # Compute loss if hasattr(model, 'hyp'): # if model has loss hyperparameters loss += compute_loss(train_out, targets, model)[1][:3].cpu() # GIoU, obj, cls # Run NMS output = non_max_suppression(inf_out, conf_thres=conf_thres, nms_thres=nms_thres) # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append(([], torch.Tensor(), torch.Tensor(), tcls)) continue if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(Path(paths[si]).stem.split('_')[-1]) box = pred[:, :4].clone() # xyxy scale_coords(imgs[si].shape[1:], box, shapes[si]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for di, d in enumerate(pred): jdict.append({ 'image_id': image_id, 'category_id': coco91class[int(d[6])], 'bbox': [floatn(x, 3) for x in box[di]], 'score': floatn(d[4], 5) }) # Clip boxes to image bounds clip_coords(pred, (height, width)) # Assign all predictions as incorrect correct = [0] * len(pred) if nl: detected = [] tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) tbox[:, [0, 2]] *= width tbox[:, [1, 3]] *= height # Search for correct predictions for i, (*pbox, pconf, pcls_conf, pcls) in enumerate(pred): # Break if all targets already located in image if len(detected) == nl: break # Continue if predicted class not among image classes if pcls.item() not in tcls: continue # Best iou, index between pred and targets m = (pcls == tcls_tensor).nonzero().view(-1) iou, bi = bbox_iou(pbox, tbox[m]).max(0) # If iou > threshold and class is correct mark as correct if iou > iou_thres and m[ bi] not in detected: # and pcls == tcls[bi]: correct[i] = 1 detected.append(m[bi]) # Append statistics (correct, conf, pcls, tcls) stats.append((correct, pred[:, 4].cpu(), pred[:, 6].cpu(), tcls)) # Compute statistics stats = [np.concatenate(x, 0) for x in list(zip(*stats))] # to numpy if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%10.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) # Save JSON if save_json and map and len(jdict): imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataset.img_files] with open('results.json', 'w') as file: json.dump(jdict, file) from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval cocoGt = COCO('../coco/annotations/instances_val2014.json' ) # initialize COCO ground truth api cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() map = cocoEval.stats[1] # update mAP to pycocotools mAP # Return results maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map, mf1, *(loss / len(dataloader)).tolist()), maps
float(ws[k]), float(hs[k])], 'score': float(scores[k]) } for k in range(det.shape[0])] result = sorted(result, key=lambda x: x['score'])[-100:] coco_result += result t5_s = time.time() print("convert to coco format uses: %.1f" % (t5_s - t3_s)) import json json.dump(coco_result, open( "experiments/{}/{}_proposal_result.json".format( pGen.name, pDataset.image_set[0]), "w"), sort_keys=True, indent=2) coco_dt = coco.loadRes(coco_result) coco_eval = COCOeval(coco, coco_dt) coco_eval.params.iouType = "bbox" coco_eval.params.maxDets = [1, 10, 100] # [100, 300, 1000] coco_eval.params.useCats = False coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() t6_s = time.time() print("coco eval uses: %.1f" % (t6_s - t5_s))
def infer(model, path, detections_file, resize, max_size, batch_size, mixed_precision=False, is_master=True, world=0, annotations=None, use_dali=True, is_validation=False, verbose=True): 'Run inference on images from path' print('model',model) backend = 'pytorch' if isinstance(model, Model) or isinstance(model, DDP) else 'tensorrt' #print("backend",backend) stride = model.module.stride if isinstance(model, DDP) else model.stride #print('!!!!!!!!model.stride:', model.stride) # Create annotations if none was provided if not annotations: annotations = tempfile.mktemp('.json') images = [{ 'id': i, 'file_name': f} for i, f in enumerate(os.listdir(path))] json.dump({ 'images': images }, open(annotations, 'w')) # TensorRT only supports fixed input sizes, so override input size accordingly if backend == 'tensorrt': max_size = max(model.input_size) # Prepare dataset if verbose: print('Preparing dataset...') data_iterator = (DaliDataIterator if use_dali else DataIterator)( path, resize, max_size, batch_size, stride, world, annotations, training=False) if verbose: print(data_iterator) # Prepare model if backend is 'pytorch': # If we are doing validation during training, # no need to register model with AMP again if not is_validation: if torch.cuda.is_available(): model = model.cuda() model = amp.initialize(model, None, opt_level = 'O2' if mixed_precision else 'O0', keep_batchnorm_fp32 = True, verbosity = 0) model.eval() if verbose: print(' backend: {}'.format(backend)) print(' device: {} {}'.format( world, 'cpu' if not torch.cuda.is_available() else 'gpu' if world == 1 else 'gpus')) print(' batch: {}, precision: {}'.format(batch_size, 'unknown' if backend is 'tensorrt' else 'mixed' if mixed_precision else 'full')) print('Running inference...') results = [] profiler = Profiler(['infer', 'fw']) with torch.no_grad(): for i, (data, ids, ratios) in enumerate(data_iterator): # Forward pass #print('start profiler') profiler.start('fw') #print("data:",data) scores, boxes, classes = model(data) profiler.stop('fw') #cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2) results.append([scores, boxes, classes, ids, ratios]) profiler.bump('infer') if verbose and (profiler.totals['infer'] > 60 or i == len(data_iterator) - 1): size = len(data_iterator.ids) msg = '[{:{len}}/{}]'.format(min((i + 1) * batch_size, size), size, len=len(str(size))) msg += ' {:.3f}s/{}-batch'.format(profiler.means['infer'], batch_size) msg += ' (fw: {:.3f}s)'.format(profiler.means['fw']) msg += ', {:.1f} im/s'.format(batch_size / profiler.means['infer']) print(msg, flush=True) profiler.reset() # Gather results from all devices if verbose: print('Gathering results...') results = [torch.cat(r, dim=0) for r in zip(*results)] if world > 1: for r, result in enumerate(results): all_result = [torch.ones_like(result, device=result.device) for _ in range(world)] torch.distributed.all_gather(list(all_result), result) results[r] = torch.cat(all_result, dim=0) if is_master: # Copy buffers back to host results = [r.cpu() for r in results] # Collect detections detections = [] processed_ids = set() for scores, boxes, classes, image_id, ratios in zip(*results): image_id = image_id.item() if image_id in processed_ids: continue processed_ids.add(image_id) keep = (scores > 0).nonzero() scores = scores[keep].view(-1) boxes = boxes[keep, :].view(-1, 4) / ratios classes = classes[keep].view(-1).int() #print('classes', classes) for score, box, cat in zip(scores, boxes, classes): x1, y1, x2, y2 = box.data.tolist() cat = cat.item() if 'annotations' in data_iterator.coco.dataset: cat = data_iterator.coco.getCatIds()[cat] #if cat !=3: #continue #print('cat',cat) detections.append({ 'image_id': image_id, 'score': score.item(), 'bbox': [x1, y1, x2 - x1 + 1, y2 - y1 + 1], 'category_id': cat }) #show_detections(detections) if detections: # Save detections if detections_file and verbose: print('Writing {}...'.format(detections_file)) detections = { 'annotations': detections } detections['images'] = data_iterator.coco.dataset['images'] if 'categories' in data_iterator.coco.dataset: detections['categories'] = [data_iterator.coco.dataset['categories']] if detections_file: json.dump(detections, open(detections_file, 'w'), indent=4) # Evaluate model on dataset if 'annotations' in data_iterator.coco.dataset: if verbose: print('Evaluating model...') with redirect_stdout(None): coco_pred = data_iterator.coco.loadRes(detections['annotations']) coco_eval = COCOeval(data_iterator.coco, coco_pred, 'bbox') coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() else: print('No detections!')
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation limit: if not 0, it's the number of images to use for evaluation """ # Pick TACO images from the dataset image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] # Get corresponding TACO image IDs. taco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): # Load image image = dataset.load_image(image_id) # Run detection t = time.time() r = model.detect([image], verbose=0)[0] # r = utils.fuse_instances(r) t_prediction += (time.time() - t) if not model.config.DETECTION_SCORE_RATIO: scores = r["scores"] else: scores = r["scores"] / (r["full_scores"][:, 0] + 0.0001) # Convert results to COCO format # Cast masks to uint8 because COCO tools errors out on bool image_results = build_coco_results(dataset, taco_image_ids[i:i + 1], r["rois"], r["class_ids"], scores, r["masks"].astype(np.uint8)) results.extend(image_results) # Load results. This modifies results with additional attributes. coco_results = coco.loadRes(results) # utils.compute_confusion_matrix(coco_results, coco) # Evaluate cocoEval = COCOeval(coco, coco_results, eval_type) cocoEval.params.imgIds = taco_image_ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start)
def compute_map(labels_and_predictions, coco_gt, use_cpp_extension=True, nms_on_tpu=True): """Use model predictions to compute mAP. The evaluation code is largely copied from the MLPerf reference implementation. While it is possible to write the evaluation as a tensor metric and use Estimator.evaluate(), this approach was selected for simplicity and ease of duck testing. Args: labels_and_predictions: A map from TPU predict method. coco_gt: ground truch COCO object. use_cpp_extension: use cocoeval C++ library. nms_on_tpu: do NMS on TPU. Returns: Evaluation result. """ predictions = [] tic = time.time() if nms_on_tpu: p = [] for i in labels_and_predictions: for j in i: p.append(np.array(j, dtype=np.float32)) predictions = np.concatenate(list(p)).reshape((-1, 7)) else: for example in labels_and_predictions: if ssd_constants.IS_PADDED in example and example[ ssd_constants.IS_PADDED]: continue htot, wtot, _ = example[ssd_constants.RAW_SHAPE] pred_box = example['pred_box'] pred_scores = example['pred_scores'] indices = example['indices'] loc, label, prob = decode_single(pred_box, pred_scores, indices, ssd_constants.OVERLAP_CRITERIA, ssd_constants.MAX_NUM_EVAL_BOXES, ssd_constants.MAX_NUM_EVAL_BOXES) for loc_, label_, prob_ in zip(loc, label, prob): # Ordering convention differs, hence [1], [0] rather than [0], [1] predictions.append([ int(example[ssd_constants.SOURCE_ID]), loc_[1] * wtot, loc_[0] * htot, (loc_[3] - loc_[1]) * wtot, (loc_[2] - loc_[0]) * htot, prob_, ssd_constants.CLASS_INV_MAP[label_] ]) toc = time.time() tf.logging.info('Prepare predictions DONE (t={:0.2f}s).'.format(toc - tic)) if coco_gt is None: coco_gt = create_coco(FLAGS.val_json_file, use_cpp_extension=use_cpp_extension) if use_cpp_extension: coco_dt = coco_gt.LoadRes(np.array(predictions, dtype=np.float32)) # copybara:strip_begin coco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iou_type='bbox') # copybara:strip_end # copybara:insert coco_eval = COCOeval(coco_gt, coco_dt, iou_type='bbox') coco_eval.Evaluate() coco_eval.Accumulate() coco_eval.Summarize() stats = coco_eval.GetStats() else: coco_dt = coco_gt.loadRes(np.array(predictions)) coco_eval = COCOeval(coco_gt, coco_dt, iouType='bbox') coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats = coco_eval.stats print('Current AP: {:.5f}'.format(stats[0])) metric_names = [ 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'ARmax1', 'ARmax10', 'ARmax100', 'ARs', 'ARm', 'ARl' ] coco_time = time.time() tf.logging.info('COCO eval DONE (t={:0.2f}s).'.format(coco_time - toc)) # Prefix with "COCO" to group in TensorBoard. return {'COCO/' + key: value for key, value in zip(metric_names, stats)}
def evaluate(self, results, metric='bbox', logger=None, jsonfile_prefix=None, classwise=False, proposal_nums=(100, 300, 1000), iou_thrs=None, metric_items=None): """Evaluation in COCO protocol. Args: results (list[list | tuple]): Testing results of the dataset. metric (str | list[str]): Metrics to be evaluated. Options are 'bbox', 'segm', 'proposal', 'proposal_fast'. logger (logging.Logger | str | None): Logger used for printing related information during evaluation. Default: None. jsonfile_prefix (str | None): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. classwise (bool): Whether to evaluating the AP for each class. proposal_nums (Sequence[int]): Proposal number used for evaluating recalls, such as recall@100, recall@1000. Default: (100, 300, 1000). iou_thrs (Sequence[float], optional): IoU threshold used for evaluating recalls/mAPs. If set to a list, the average of all IoUs will also be computed. If not specified, [0.50, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] will be used. Default: None. metric_items (list[str] | str, optional): Metric items that will be returned. If not specified, ``['AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ]`` will be used when ``metric=='proposal'``, ``['mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l']`` will be used when ``metric=='bbox' or metric=='segm'``. Returns: dict[str, float]: COCO style evaluation metric. """ metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in metrics: if metric not in allowed_metrics: raise KeyError(f'metric {metric} is not supported') if iou_thrs is None: iou_thrs = np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) if metric_items is not None: if not isinstance(metric_items, list): metric_items = [metric_items] result_files, tmp_dir = self.format_results(results, jsonfile_prefix) eval_results = OrderedDict() cocoGt = self.coco for metric in metrics: msg = f'Evaluating {metric}...' if logger is None: msg = '\n' + msg print_log(msg, logger=logger) if metric == 'proposal_fast': ar = self.fast_eval_recall(results, proposal_nums, iou_thrs, logger='silent') log_msg = [] for i, num in enumerate(proposal_nums): eval_results[f'AR@{num}'] = ar[i] log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') log_msg = ''.join(log_msg) print_log(log_msg, logger=logger) continue if metric not in result_files: raise KeyError(f'{metric} is not in results') try: cocoDt = cocoGt.loadRes(result_files[metric]) except IndexError: print_log('The testing results of the whole dataset is empty.', logger=logger, level=logging.ERROR) break iou_type = 'bbox' if metric == 'proposal' else metric cocoEval = COCOeval(cocoGt, cocoDt, iou_type) cocoEval.params.catIds = self.cat_ids cocoEval.params.imgIds = self.img_ids cocoEval.params.maxDets = list(proposal_nums) cocoEval.params.iouThrs = iou_thrs # mapping of cocoEval.stats coco_metric_names = { 'mAP': 0, 'mAP_50': 1, 'mAP_75': 2, 'mAP_s': 3, 'mAP_m': 4, 'mAP_l': 5, 'AR@100': 6, 'AR@300': 7, 'AR@1000': 8, 'AR_s@1000': 9, 'AR_m@1000': 10, 'AR_l@1000': 11 } if metric_items is not None: for metric_item in metric_items: if metric_item not in coco_metric_names: raise KeyError( f'metric item {metric_item} is not supported') if metric == 'proposal': cocoEval.params.useCats = 0 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() if metric_items is None: metric_items = [ 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ] for item in metric_items: val = float( f'{cocoEval.stats[coco_metric_names[item]]:.3f}') eval_results[item] = val else: cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() if classwise: # Compute per-category AP # Compute per-category AP # from https://github.com/facebookresearch/detectron2/ precisions = cocoEval.eval['precision'] # precision: (iou, recall, cls, area range, max dets) assert len(self.cat_ids) == precisions.shape[2] results_per_category = [] for idx, catId in enumerate(self.cat_ids): # area range index 0: all area ranges # max dets index -1: typically 100 per image nm = self.coco.loadCats(catId)[0] precision = precisions[:, :, idx, 0, -1] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (f'{nm["name"]}', f'{float(ap):0.3f}')) num_columns = min(6, len(results_per_category) * 2) results_flatten = list( itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest(*[ results_flatten[i::num_columns] for i in range(num_columns) ]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) print_log('\n' + table.table, logger=logger) if metric_items is None: metric_items = [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' ] for metric_item in metric_items: key = f'{metric}_{metric_item}' val = float( f'{cocoEval.stats[coco_metric_names[metric_item]]:.3f}' ) eval_results[key] = val ap = cocoEval.stats[:6] eval_results[f'{metric}_mAP_copypaste'] = ( f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' f'{ap[4]:.3f} {ap[5]:.3f}') if tmp_dir is not None: tmp_dir.cleanup() return eval_results
def test( data, weights=None, batch_size=32, imgsz=640, conf_thres=0.001, iou_thres=0.6, # for NMS save_json=False, single_cls=False, augment=False, verbose=False, model=None, dataloader=None, save_dir=Path(''), # for saving images save_txt=False, # for auto-labelling save_hybrid=False, # for hybrid auto-labelling save_conf=False, # save auto-label confidences plots=True, wandb_logger=None, compute_loss=None, half_precision=True, is_coco=False): # Initialize/load model and set device training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly set_logging() device = select_device(opt.device, batch_size=batch_size) # Directories save_dir = Path( increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir( parents=True, exist_ok=True) # make dir # Load model model = attempt_load(weights, map_location=device) # load FP32 model gs = max(int(model.stride.max()), 32) # grid size (max stride) imgsz = check_img_size(imgsz, s=gs) # check img_size # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 # if device.type != 'cpu' and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) # Half half = device.type != 'cpu' and half_precision # half precision only supported on CUDA if half: model.half() # Configure model.eval() if isinstance(data, str): is_coco = data.endswith('coco.yaml') with open(data) as f: data = yaml.load(f, Loader=yaml.SafeLoader) check_dataset(data) # check nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Logging log_imgs = 0 if wandb_logger and wandb_logger.wandb: log_imgs = min(wandb_logger.log_imgs, 100) # Dataloader if not training: if device.type != 'cpu': model( torch.zeros(1, 3, imgsz, imgsz).to(device).type_as( next(model.parameters()))) # run once task = opt.task if opt.task in ( 'train', 'val', 'test') else 'val' # path to train/val/test images dataloader = create_dataloader(data[task], imgsz, batch_size, gs, opt, pad=0.5, rect=True, prefix=colorstr(f'{task}: '))[0] seen = 0 confusion_matrix = ConfusionMatrix(nc=nc) names = { k: v for k, v in enumerate( model.names if hasattr(model, 'names') else model.module.names) } coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', '[email protected]', '[email protected]:.95') p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class, wandb_images = [], [], [], [], [] for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): img = img.to(device, non_blocking=True) img = img.half() if half else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width with torch.no_grad(): # Run model t = time_synchronized() out, train_out = model( img, augment=augment) # inference and training outputs t0 += time_synchronized() - t # Compute loss if compute_loss: loss += compute_loss([x.float() for x in train_out], targets)[1][:3] # box, obj, cls # Run NMS targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels lb = [targets[targets[:, 0] == i, 1:] for i in range(nb) ] if save_hybrid else [] # for autolabelling t = time_synchronized() out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=True) t1 += time_synchronized() - t # Statistics per image for si, pred in enumerate(out): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class path = Path(paths[si]) seen += 1 if len(pred) == 0: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Predictions predn = pred.clone() scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0 ]] # normalization gain whwh for *xyxy, conf, cls in predn.tolist(): xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f: f.write(('%g ' * len(line)).rstrip() % line + '\n') # W&B logging - Media Panel Plots if len( wandb_images ) < log_imgs and wandb_logger.current_epoch > 0: # Check for test operation if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0: box_data = [{ "position": { "minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3] }, "class_id": int(cls), "box_caption": "%s %.3f" % (names[cls], conf), "scores": { "class_score": conf }, "domain": "pixel" } for *xyxy, conf, cls in pred.tolist()] boxes = { "predictions": { "box_data": box_data, "class_labels": names } } # inference-space wandb_images.append( wandb_logger.wandb.Image(img[si], boxes=boxes, caption=path.name)) wandb_logger.log_training_progress( predn, path, names) if wandb_logger and wandb_logger.wandb_run else None # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int( path.stem) if path.stem.isnumeric() else path.stem box = xyxy2xywh(predn[:, :4]) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({ 'image_id': image_id, 'category_id': coco91class[int(p[5])] if is_coco else int(p[5]), 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5) }) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels if plots: confusion_matrix.process_batch( predn, torch.cat((labels[:, 0:1], tbox), 1)) # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view( -1) # prediction indices pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view( -1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(predn[pi, :4], tbox[ti]).max( 1) # best ious, indices # Append detections detected_set = set() for j in (ious > iouv[0]).nonzero(as_tuple=False): d = ti[i[j]] # detected target if d.item() not in detected_set: detected_set.add(d.item()) detected.append(d) correct[ pi[j]] = ious[j] > iouv # iou_thres is 1xn if len( detected ) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append( (correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Plot images if plots and batch_i < 3: f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start() f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions Thread(target=plot_images, args=(img, output_to_target(out), paths, f, names), daemon=True).start() # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%12i' * 2 + '%12.3g' * 4 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple if not training: print( 'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb_logger and wandb_logger.wandb: val_batches = [ wandb_logger.wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg')) ] wandb_logger.log({"Validation": val_batches}) if wandb_images: wandb_logger.log({"Bounding Box Debugger/Images": wandb_images}) # Save JSON if save_json and len(jdict): w = Path(weights[0] if isinstance(weights, list) else weights ).stem if weights is not None else '' # weights anno_json = '../coco/annotations/instances_val2017.json' # annotations json pred_json = str(save_dir / f"{w}_predictions.json") # predictions json print('\nEvaluating pycocotools mAP... saving %s...' % pred_json) with open(pred_json, 'w') as f: json.dump(jdict, f) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval anno = COCO(anno_json) # init annotations api pred = anno.loadRes(pred_json) # init predictions api eval = COCOeval(anno, pred, 'bbox') if is_coco: eval.params.imgIds = [ int(Path(x).stem) for x in dataloader.dataset.img_files ] # image IDs to evaluate eval.evaluate() eval.accumulate() eval.summarize() map, map50 = eval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: print(f'pycocotools unable to run: {e}') # Return results model.float() # for training if not training: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' print(f"Results saved to {save_dir}{s}") maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(generator, model, anchors, json_path, imsize=448, threshold=0.5): """ Use the pycocotools to evaluate a COCO model on a dataset. Args generator : The generator for generating the evaluation data. model : The model to evaluate. threshold : The score threshold to use. """ # start collecting results import pickle if os.path.exists('coco_eval_temp.pk'): results, image_ids = pickle.load(open('coco_eval_temp.pk', 'rb')) else: results = [] image_ids = [] for index in range(generator.size()): # if index % 50 == 0: # print() print(index, end='\r') image = generator.load_image(index) image, scale = resize_image(image, 360, imsize) image = np.expand_dims(image, 0) boxes = get_yolo_boxes(model, image, imsize, imsize, anchors, 0.5, 0.5, preprocess=True)[0] boxes, scores, labels = boundbox2cocobox(boxes, scale) # assert len(boxes) > 0 # compute predicted labels and scores image_id = int(os.path.split(generator.instances[index]['filename'])[-1][:-4]) for box, score, label in zip(boxes, scores, labels): # scores are sorted, so we can break if score < threshold: break # append detection for each positively labeled class image_result = { 'image_id': image_id, 'category_id': label_to_coco_label(label), # todo: 'score': float(score), 'bbox': box, } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(image_id) with open('coco_eval_temp.pk', 'wb') as wr: pickle.dump([results, image_ids], wr) if not len(results): return import json # write output json.dump(results, open('{}_bbox_results.json'.format('val2017'), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format('val2017'), 'w'), indent=4) # load results in COCO evaluation tool coco_true = COCO(json_path) coco_pred = coco_true.loadRes('{}_bbox_results.json'.format('val2017')) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval.stats
def run( data, weights=None, # model.pt path(s) batch_size=32, # batch size imgsz=640, # inference size (pixels) conf_thres=0.001, # confidence threshold iou_thres=0.6, # NMS IoU threshold task='val', # train, val, test, speed or study device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu workers=8, # max dataloader workers (per RANK in DDP mode) single_cls=False, # treat as single-class dataset augment=False, # augmented inference verbose=False, # verbose output save_txt=False, # save results to *.txt save_hybrid=False, # save label+prediction hybrid results to *.txt save_conf=False, # save confidences in --save-txt labels save_json=False, # save a COCO-JSON results file project=ROOT / 'runs/val', # save to project/name name='exp', # save to project/name exist_ok=False, # existing project/name ok, do not increment half=True, # use FP16 half-precision inference dnn=False, # use OpenCV DNN for ONNX inference model=None, dataloader=None, save_dir=Path(''), plots=True, callbacks=Callbacks(), compute_loss=None, ): # Initialize/load model and set device training = model is not None if training: # called by train.py device, pt, jit, engine = next(model.parameters( )).device, True, False, False # get model device, PyTorch model half &= device.type != 'cpu' # half precision only supported on CUDA model.half() if half else model.float() else: # called directly device = select_device(device, batch_size=batch_size) # Directories save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir( parents=True, exist_ok=True) # make dir # Load model model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data) stride, pt, jit, onnx, engine = model.stride, model.pt, model.jit, model.onnx, model.engine imgsz = check_img_size(imgsz, s=stride) # check image size half &= ( pt or jit or onnx or engine ) and device.type != 'cpu' # FP16 supported on limited backends with CUDA if pt or jit: model.model.half() if half else model.model.float() elif engine: batch_size = model.batch_size if model.trt_fp16_input != half: LOGGER.info('model ' + ('requires' if model. trt_fp16_input else 'incompatible with') + ' --half. Adjusting automatically.') half = model.trt_fp16_input else: half = False batch_size = 1 # export.py models default to batch-size 1 device = torch.device('cpu') LOGGER.info( f'Forcing --batch-size 1 square inference shape(1,3,{imgsz},{imgsz}) for non-PyTorch backends' ) # Data data = check_dataset(data) # check # Configure model.eval() is_coco = isinstance(data.get('val'), str) and data['val'].endswith( 'coco/val2017.txt') # COCO dataset nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Dataloader if not training: model.warmup(imgsz=(1 if pt else batch_size, 3, imgsz, imgsz), half=half) # warmup pad = 0.0 if task in ('speed', 'benchmark') else 0.5 rect = False if task == 'benchmark' else pt # square inference for benchmarks task = task if task in ( 'train', 'val', 'test') else 'val' # path to train/val/test images dataloader = create_dataloader(data[task], imgsz, batch_size, stride, single_cls, pad=pad, rect=rect, workers=workers, prefix=colorstr(f'{task}: '))[0] seen = 0 confusion_matrix = ConfusionMatrix(nc=nc) names = { k: v for k, v in enumerate( model.names if hasattr(model, 'names') else model.module.names) } class_map = coco80_to_coco91_class() if is_coco else list(range(1000)) s = ('%20s' + '%11s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', '[email protected]', '[email protected]:.95') dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0, 0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] pbar = tqdm(dataloader, desc=s, bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}') # progress bar for batch_i, (im, targets, paths, shapes) in enumerate(pbar): t1 = time_sync() if pt or jit or engine: im = im.to(device, non_blocking=True) targets = targets.to(device) im = im.half() if half else im.float() # uint8 to fp16/32 im /= 255 # 0 - 255 to 0.0 - 1.0 nb, _, height, width = im.shape # batch size, channels, height, width t2 = time_sync() dt[0] += t2 - t1 # Inference out, train_out = model(im) if training else model( im, augment=augment, val=True) # inference, loss outputs dt[1] += time_sync() - t2 # Loss if compute_loss: loss += compute_loss([x.float() for x in train_out], targets)[1] # box, obj, cls # NMS targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling t3 = time_sync() out = non_max_suppression(out, conf_thres, iou_thres, labels=lb, multi_label=True, agnostic=single_cls) dt[2] += time_sync() - t3 # Metrics for si, pred in enumerate(out): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class path, shape = Path(paths[si]), shapes[si][0] seen += 1 if len(pred) == 0: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Predictions if single_cls: pred[:, 5] = 0 predn = pred.clone() scale_coords(im[si].shape[1:], predn[:, :4], shape, shapes[si][1]) # native-space pred # Evaluate if nl: tbox = xywh2xyxy(labels[:, 1:5]) # target boxes scale_coords(im[si].shape[1:], tbox, shape, shapes[si][1]) # native-space labels labelsn = torch.cat((labels[:, 0:1], tbox), 1) # native-space labels correct = process_batch(predn, labelsn, iouv) if plots: confusion_matrix.process_batch(predn, labelsn) else: correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool) stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # (correct, conf, pcls, tcls) # Save/log if save_txt: save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / (path.stem + '.txt')) if save_json: save_one_json(predn, jdict, path, class_map) # append to COCO-JSON dictionary callbacks.run('on_val_image_end', pred, predn, path, names, im[si]) # Plot images if plots and batch_i < 3: f = save_dir / f'val_batch{batch_i}_labels.jpg' # labels Thread(target=plot_images, args=(im, targets, paths, f, names), daemon=True).start() f = save_dir / f'val_batch{batch_i}_pred.jpg' # predictions Thread(target=plot_images, args=(im, output_to_target(out), paths, f, names), daemon=True).start() # Compute metrics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%11i' * 2 + '%11.3g' * 4 # print format LOGGER.info(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): for i, c in enumerate(ap_class): LOGGER.info(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in dt) # speeds per image if not training: shape = (batch_size, 3, imgsz, imgsz) LOGGER.info( f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}' % t) # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) callbacks.run('on_val_end') # Save JSON if save_json and len(jdict): w = Path(weights[0] if isinstance(weights, list) else weights ).stem if weights is not None else '' # weights anno_json = str( Path(data.get('path', '../coco')) / 'annotations/instances_val2017.json') # annotations json pred_json = str(save_dir / f"{w}_predictions.json") # predictions json LOGGER.info(f'\nEvaluating pycocotools mAP... saving {pred_json}...') with open(pred_json, 'w') as f: json.dump(jdict, f) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb check_requirements(['pycocotools']) from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval anno = COCO(anno_json) # init annotations api pred = anno.loadRes(pred_json) # init predictions api eval = COCOeval(anno, pred, 'bbox') if is_coco: eval.params.imgIds = [ int(Path(x).stem) for x in dataloader.dataset.im_files ] # image IDs to evaluate eval.evaluate() eval.accumulate() eval.summarize() map, map50 = eval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: LOGGER.info(f'pycocotools unable to run: {e}') # Return results model.float() # for training if not training: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate(self, model): """ COCO average precision (AP) Evaluation. Iterate inference on the test dataset and the results are evaluated by COCO API. Args: model : model object Returns: ap50_95 (float) : calculated COCO AP for IoU=50:95 ap50 (float) : calculated COCO AP for IoU=50 """ model.eval() ids = [] data_dict = [] num_images = len(self.dataset) print('total number of images: %d' % (num_images)) # start testing for index in range(num_images): # all the data in val2017 if index % 500 == 0: print('[Eval: %d / %d]' % (index, num_images)) # load an image img, id_ = self.dataset.pull_image(index) h, w, _ = img.shape size = np.array([[w, h, w, h]]) # preprocess x, _, _, scale, offset = self.transform(img) x = x.unsqueeze(0).to(self.device) id_ = int(id_) ids.append(id_) # inference with torch.no_grad(): outputs = model(x) bboxes, scores, cls_inds = outputs # map the boxes to original image bboxes -= offset bboxes /= scale bboxes *= size for i, box in enumerate(bboxes): x1 = float(box[0]) y1 = float(box[1]) x2 = float(box[2]) y2 = float(box[3]) label = self.dataset.class_ids[int(cls_inds[i])] bbox = [x1, y1, x2 - x1, y2 - y1] score = float(scores[i]) # object score * class score A = { "image_id": id_, "category_id": label, "bbox": bbox, "score": score } # COCO json format data_dict.append(A) annType = ['segm', 'bbox', 'keypoints'] # Evaluate the Dt (detection) json comparing with the ground truth if len(data_dict) > 0: print('evaluating ......') cocoGt = self.dataset.coco # workaround: temporarily write data to json file because pycocotools can't process dict in py36. if self.testset: json.dump(data_dict, open('coco_test-dev.json', 'w')) cocoDt = cocoGt.loadRes('coco_test-dev.json') return -1, -1 else: _, tmp = tempfile.mkstemp() json.dump(data_dict, open(tmp, 'w')) cocoDt = cocoGt.loadRes(tmp) cocoEval = COCOeval(self.dataset.coco, cocoDt, annType[1]) cocoEval.params.imgIds = ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() ap50_95, ap50 = cocoEval.stats[0], cocoEval.stats[1] print('ap50_95 : ', ap50_95) print('ap50 : ', ap50) self.map = ap50_95 self.ap50_95 = ap50_95 self.ap50 = ap50 return ap50, ap50_95 else: return 0, 0
def evaluate(config_file="configs/COCO-Detection/yolov5-small.yaml", batch_size=16, data="data/coco2017.yaml", image_size=640, weights=None, confidence_thresholds=0.001, iou_thresholds=0.6, save_json=False, merge=False, augment=False, verbose=False, save_txt=False, model=None, dataloader=None): with open(data) as f: data_dict = yaml.load(f, Loader=yaml.FullLoader) number_classes, names = int( data_dict["number_classes"]), data_dict["names"] assert len( names ) == number_classes, f"{len(names)} names found for nc={number_classes} dataset in {data}" # Initialize/load model and set device training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly device = select_device(args.device, batch_size=args.batch_size) if save_txt: if os.path.exists("outputs"): shutil.rmtree("outputs") # delete output folder os.makedirs("outputs") # make new output folder # Create model model = YOLO(config_file=config_file, number_classes=number_classes).to(device) # Load model model.load_state_dict(torch.load(weights)["state_dict"]) model.float() model.fuse() model.eval() # Half half = device.type != "cpu" # half precision only supported on CUDA if half: model.half() # Configure model.eval() iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Dataloader if not training: image = torch.zeros((1, 3, image_size, image_size), device=device) # init image _ = model(image.half() if half else image ) if device.type != "cpu" else None # run once dataroot = data_dict["test"] if data_dict["test"] else data_dict[ "val"] # path to val/test images dataset, dataloader = create_dataloader(dataroot=dataroot, image_size=image_size, batch_size=batch_size, hyper_parameters=None, augment=False, cache=False, rect=True) seen = 0 coco91class = coco80_to_coco91_class() context = f"{'Class':>20}{'Images':>12}{'Targets':>12}{'P':>12}{'R':>12}{'[email protected]':>12}{'[email protected]:.95':>12}" p, r, f1, mp, mr, map50, map, inference_time, nms_time = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] for _, (image, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=context)): image = image.to(device, non_blocking=True) image = image.half() if half else image.float() # uint8 to fp16/32 image /= 255.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = image.shape # batch size, channels, height, width whwh = torch.Tensor([width, height, width, height]).to(device) # Disable gradients with torch.no_grad(): # Run model t = time_synchronized() prediction, outputs = model( image, augment=augment) # inference and training outputs inference_time += time_synchronized() - t # Compute loss if training: # if model has loss hyper parameters loss += compute_loss([x.float() for x in outputs], targets, model)[1][:3] # GIoU, obj, cls # Run NMS t = time_synchronized() prediction = non_max_suppression( prediction=prediction, confidence_thresholds=confidence_thresholds, iou_thresholds=iou_thresholds, merge=merge, classes=None, agnostic=False) nms_time += time_synchronized() - t # Statistics per image for si, pred in enumerate(prediction): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0 ]] # normalization gain whwh txt_path = os.path.join("outputs", paths[si].split("/")[-1][:-4]) pred[:, :4] = scale_coords(image[si].shape[1:], pred[:, :4], shapes[si][0], shapes[si][1]) # to original for *xyxy, conf, cls in pred: xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh with open(txt_path + ".txt", "a") as f: f.write( ("%g " * 5 + "\n") % (cls, *xywh)) # label format # Clip boxes to image bounds clip_coords(pred, (height, width)) # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = si[:-4] box = pred[:, :4].clone() # xyxy scale_coords(image[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({ "image_id": int(image_id) if image_id.isnumeric() else image_id, "category_id": coco91class[int(p[5])], "bbox": [round(x, 3) for x in b], "score": round(p[4], 5) }) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view( -1) # prediction indices pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view( -1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(pred[pi, :4], tbox[ti]).max( 1) # best ious, indices # Append detections for j in (ious > iouv[0]).nonzero(as_tuple=False): d = ti[i[j]] # detected target if d not in detected: detected.append(d) correct[ pi[j]] = ious[j] > iouv # iou_thres is 1xn if len( detected ) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append( (correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): p, r, ap, f1, ap_class = ap_per_class(*stats) p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean( 1) # [P, R, [email protected], [email protected]:0.95] mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype( np.int64), minlength=number_classes) # number of targets per class else: nt = torch.zeros(1) # Print results print( f"{'all':>20}{seen:>12}{nt.sum():>12}{mp:>12.3f}{mr:>12.3f}{map50:>12.3f}{map:>12.3f}" ) # Print results per class if verbose: for i, c in enumerate(ap_class): print( f"{names[c]:>20}{seen:>12}{nt[c]:>12}{p[i]:>12.3f}{r[i]:>12.3f}{ap50[i]:>12.3f}{ap[i]:>12.3f}" ) # Print speeds if not training: print( "Speed: " f"{inference_time / seen * 1000:.1f}/" f"{nms_time / seen * 1000:.1f}/" f"{(inference_time + nms_time) / seen * 1000:.1f} ms " f"inference/NMS/total per {image_size}x{image_size} image at batch-size {batch_size}" ) # Save JSON if save_json and len(jdict): f = f"detections_val2017_{weights.split('/')[-1].replace('.pth', '')}_results.json" print(f"\nCOCO mAP with pycocotools... saving {f}...") with open(f, "w") as file: json.dump(jdict, file) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval imgIds = [ int(x.split("/")[-1][:-4]) for x in dataloader.dataset.image_files ] cocoGt = COCO( glob.glob("data/coco2017/annotations/instances_val*.json")[0]) cocoDt = cocoGt.loadRes(f) # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, "bbox") cocoEval.params.imgIds = imgIds # image IDs to evaluate cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() map, map50 = cocoEval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except Exception as e: print(f"ERROR: pycocotools unable to run: {e}") # Return results model.float() # for training maps = np.zeros(int(data_dict["number_classes"])) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps
def test( data, weights=None, batch_size=16, imgsz=640, conf_thres=0.001, iou_thres=0.6, # for NMS save_json=False, verbose=False, model=None, dataloader=None, logdir='./runs', merge=False): # Initialize/load model and set device if model is None: training = False device = torch_utils.select_device(opt.device, batch_size=batch_size) # Remove previous for f in glob.glob(os.path.join(logdir, 'test_batch*.jpg')): os.remove(f) # Load model model = torch.load( weights, map_location=device)['model'].float() # load to FP32 torch_utils.model_info(model) model.fuse() model.to(device) # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 # if device.type != 'cpu' and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) else: # called by train.py training = True device = next(model.parameters()).device # get model device # Half half = device.type != 'cpu' and torch.cuda.device_count( ) == 1 # half precision only supported on single-GPU half = False if half: model.half() # to FP16 # Configure model.eval() with open(data) as f: data = yaml.load(f, Loader=yaml.FullLoader) # model dict nc = int(data['num_classes']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() losser = YoloLoss(model) # Dataloader if dataloader is None: # not training merge = opt.merge # use Merge NMS img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img _ = model(img.half() if half else img ) if device.type != 'cpu' else None # run once path = data['test'] if opt.task == 'test' else data[ 'val'] # path to val/test images dataloader = kitti.create_dataloader(path, imgsz, batch_size, int(max(model.stride)), config=None, augment=False, cache=False, pad=0.5, rect=True)[0] seen = 0 names = data['names'] kitti8class = data_utils.kitti8_classes() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', '[email protected]:.95') p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (img, targets, paths, shapes) in enumerate(tqdm.tqdm(dataloader, desc=s)): targets.delete_by_mask() targets.to_float32() targ = ParamList(targets.size, True) targ.copy_from(targets) img_id = targets.get_field('img_id') classes = targets.get_field('class') bboxes = targets.get_field('bbox') targets = torch.cat( [img_id.unsqueeze(-1), classes.unsqueeze(-1), bboxes], dim=-1) img = img.to(device) img = img.half() if half else img.float() # uint8 to fp16/32 # img /= 1.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width whwh = torch.Tensor([width, height, width, height]).to(device) # Disable gradients with torch.no_grad(): # Run model t = torch_utils.time_synchronized() inf_out, train_out = model(img) # inference and training outputs t0 += torch_utils.time_synchronized() - t # Compute loss if training: # if model has loss hyperparameters # loss += calc_loss([x.float() for x in train_out], targets, model)[1][:3] # GIoU, obj, cls loss += losser([x.float() for x in train_out], targ)[1][:3] # Run NMS t = torch_utils.time_synchronized() output = postprocess.apply_nms(inf_out, nc, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge) t1 += torch_utils.time_synchronized() - t # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file # with open('test.txt', 'a') as file: # [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred] # Clip boxes to image bounds utils.clip_coords(pred, (height, width)) # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(Path(paths[si]).stem.split('_')[-1]) box = pred[:, :4].clone() # xyxy utils.scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape box = data_utils.xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({ 'image_id': image_id, 'category_id': kitti8class[int(p[5])], 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5) }) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = data_utils.xywh2xyxy(labels[:, 1:5]) * whwh # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view( -1) # prediction indices pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view( -1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = metrics_utils.box_iou( pred[pi, :4], tbox[ti]).max(1) # best ious, indices # Append detections for j in (ious > iouv[0]).nonzero(as_tuple=False): d = ti[i[j]] # detected target if d not in detected: detected.append(d) correct[ pi[j]] = ious[j] > iouv # iou_thres is 1xn if len( detected ) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append( (correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Plot images if batch_i < 1: f = os.path.join(logdir, 'test_batch%g_gt.jpg' % batch_i) # filename visual_utils.plot_images(img, targets, paths, f, names) # ground truth f = os.path.join(logdir, 'test_batch%g_pred.jpg' % batch_i) visual_utils.plot_images(img, utils.output_to_target( output, width, height), paths, f, names) # predictions # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats): p, r, ap, f1, ap_class = metrics_utils.ap_per_class(*stats) p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean( 1) # [P, R, [email protected], [email protected]:0.95] mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%12.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple if not training: print( 'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Save JSON if save_json and map50 and len(jdict): imgIds = [ int(Path(x).stem.split('_')[-1]) for x in dataloader.dataset.img_files ] f = 'detections_val2017_%s_results.json' % \ (weights.split(os.sep)[-1].replace('.pt', '') if weights else '') # filename print('\nCOCO mAP with pycocotools... saving %s...' % f) with open(f, 'w') as file: json.dump(jdict, file) try: from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb cocoGt = COCO( glob.glob('../coco/annotations/instances_val*.json') [0]) # initialize COCO ground truth api cocoDt = cocoGt.loadRes(f) # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # image IDs to evaluate cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() map, map50 = cocoEval.stats[: 2] # update results ([email protected]:0.95, [email protected]) except: print( 'WARNING: pycocotools must be installed with numpy==1.17 to run correctly. ' 'See https://github.com/cocodataset/cocoapi/issues/356') # Return results model.float() # for training maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(generator, model, threshold=0.05): """ Use the pycocotools to evaluate a COCO model on a dataset. Args generator : The generator for generating the evaluation data. model : The model to evaluate. threshold : The score threshold to use. """ # start collecting results results = [] image_ids = [] for index in progressbar.progressbar(range(generator.size()), prefix='COCO evaluation: '): image = generator.load_image(index) image = generator.preprocess_image(image) image, scale = generator.resize_image(image) if keras.backend.image_data_format() == 'channels_first': image = image.transpose((2, 0, 1)) # run network boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0)) # correct boxes for image scale boxes /= scale # change to (x, y, w, h) (MS COCO standard) boxes[:, :, 2] -= boxes[:, :, 0] boxes[:, :, 3] -= boxes[:, :, 1] # compute predicted labels and scores for box, score, label in zip(boxes[0], scores[0], labels[0]): # scores are sorted, so we can break if score < threshold: break # append detection for each positively labeled class image_result = { 'image_id' : generator.image_ids[index], 'category_id' : generator.label_to_coco_label(label), 'score' : float(score), 'bbox' : box.tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(generator.image_ids[index]) if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = generator.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval.stats
def main(random_seed, test_on_gt, only_test, overfit): random.seed(random_seed) np.random.seed(random_seed) torch.manual_seed(random_seed) torch.cuda.manual_seed_all(random_seed) n_epochs = 3 lr = 1e-2 wd = 0 lr_scheduler = True train_db = JointCocoTasks() network = JointClassifier() optimizer = SGD(network.parameters(), lr=lr, weight_decay=wd) experiment = JointClassifierExperiment( network=network, optimizer=optimizer, dataset=train_db, tensorboard=True, seed=random_seed, ) train_folder = "ablation-joint-classifier-seed:{s}".format(s=random_seed) folder = os.path.join(SAVING_DIRECTORY, train_folder) mkdir_p(folder) if not only_test: experiment.train_n_epochs(n_epochs, overfit=overfit, lr_scheduler=lr_scheduler) torch.save(network.state_dict(), os.path.join(folder, "model.mdl")) else: network.load_state_dict(torch.load(os.path.join(folder, "model.mdl"))) for task_number in TASK_NUMBERS: if test_on_gt: test_db = CocoTasksTestGT(task_number) else: test_db = CocoTasksTest(task_number) print("testing task {}".format(task_number), "---------------------") # test_model detections = experiment.do_test(test_db, task_number=task_number) detections_file_name = "detections_tn:{}_tgt:{}.json".format( task_number, test_on_gt) # save detections with open(os.path.join(folder, detections_file_name), "w") as f: json.dump(detections, f) # perform evaluation with redirect_stdout(open(os.devnull, "w")): gtCOCO = test_db.task_coco dtCOCO = gtCOCO.loadRes(os.path.join(folder, detections_file_name)) cocoEval = COCOeval(gtCOCO, dtCOCO, "bbox") cocoEval.params.catIds = 1 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("mAP:\t\t %1.6f" % cocoEval.stats[0]) print("[email protected]:\t\t %1.6f" % cocoEval.stats[1]) # save evaluation performance result_file_name = "result_tn:{}_tgt:{}.txt".format( task_number, test_on_gt) with open(os.path.join(folder, result_file_name), "w") as f: f.write("%1.6f, %1.6f" % (cocoEval.stats[0], cocoEval.stats[1]))
res_fn = '/home/admin/jupyter/wei/mmdetection-master/f_r50_v0.6_val_8.pkl.bbox.json' ann_fn = '/home/admin/jupyter/wei/mmdetection-master/data/coco/annotations/instances_val2017_11cls.json' root = 'jsons_from_v0.7/' gt_path = root + 'gt_cls11map6_val2017.json' res_path = root + 'res_cls11map6_v0.6.json' gt_11cls = json.load(open(ann_fn, 'r')) res_11cls = json.load(open(res_fn, 'r')) for i in range(len(res_11cls)): cat_id = res_11cls[i]['category_id'] res_11cls[i]['category_id'] = cat_id11map6[cat_id - 1] json.dump(cls11map6(gt_11cls), open(gt_path, 'w'), ensure_ascii=False, indent=2) json.dump(res_11cls, open(res_path, 'w'), ensure_ascii=False, indent=2) coco = COCO(gt_path) coco_dets = coco.loadRes(res_path) cocoEval = COCOeval(coco, coco_dets, 'bbox') img_ids = coco.getImgIds() cocoEval.params.imgIds = img_ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() table_classwise(cocoEval, coco)
def evaluate_workflow(coco_eval: COCOeval, cat_ids: List[int], cat_names: List[str], out_dir: str) -> Dict[str, float]: """Execute evaluation.""" n_tit = 12 # number of evaluation titles n_cls = len(cat_ids) # 10/8 classes for BDD100K detection/tracking n_thr = 10 # [.5:.05:.95] T=10 IoU thresholds for evaluation n_rec = 101 # [0:.01:1] R=101 recall thresholds for evaluation n_area = 4 # A=4 object area ranges for evaluation n_mdet = 3 # [1 10 100] M=3 thresholds on max detections per image eval_param = { "params": { "imgIds": [], "catIds": [], "iouThrs": np.linspace( 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05) + 1), endpoint=True, ).tolist(), "recThrs": np.linspace( 0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01) + 1), endpoint=True, ).tolist(), "maxDets": [1, 10, 100], "areaRng": [ [0**2, 1e5**2], [0**2, 32**2], [32**2, 96**2], [96**2, 1e5**2], ], "useSegm": 0, "useCats": 1, }, "date": datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3], "counts": [n_thr, n_rec, n_cls, n_area, n_mdet], "precision": -np.ones( (n_thr, n_rec, n_cls, n_area, n_mdet), order="F"), "recall": -np.ones((n_thr, n_cls, n_area, n_mdet), order="F"), } stats_all = -np.ones((n_cls, n_tit)) for i, (cat_id, cat_name) in enumerate(zip(cat_ids, cat_names)): print("\nEvaluate category: %s" % cat_name) coco_eval.params.catIds = [cat_id] # coco_eval.params.useSegm = ann_type == "segm" coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats_all[i, :] = coco_eval.stats eval_param["precision"][:, :, i, :, :] = coco_eval.eval["precision"].reshape( (n_thr, n_rec, n_area, n_mdet)) eval_param["recall"][:, i, :, :] = coco_eval.eval["recall"].reshape( (n_thr, n_area, n_mdet)) # Print evaluation results stats = np.zeros((n_tit, 1)) print("\nOverall performance") coco_eval.eval = eval_param coco_eval.summarize() for i in range(n_tit): column = stats_all[:, i] if len(column > -1) == 0: stats[i] = -1 else: stats[i] = np.mean(column[column > -1], axis=0) score_titles = [ "AP", "AP_50", "AP_75", "AP_small", "AP_medium", "AP_large", "AR_max_1", "AR_max_10", "AR_max_100", "AR_small", "AR_medium", "AR_large", ] scores: Dict[str, float] = {} for title, stat in zip(score_titles, stats): scores[title] = stat.item() if out_dir != "none": write_eval(out_dir, scores, eval_param) return scores
def test(data, weights=None, batch_size=16, imgsz=640, conf_thres=0.001, iou_thres=0.6, # for NMS save_json=False, single_cls=False, augment=False, verbose=False, model=None, dataloader=None, save_dir='', merge=False, save_txt=False): # Initialize/load model and set device training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly set_logging() device = select_device(opt.device, batch_size=batch_size) merge, save_txt = opt.merge, opt.save_txt # use Merge NMS, save *.txt labels if save_txt: out = Path('inference/output') if os.path.exists(out): shutil.rmtree(out) # delete output folder os.makedirs(out) # make new output folder # Remove previous for f in glob.glob(str(Path(save_dir) / 'test_batch*.jpg')): os.remove(f) # Load model model = attempt_load(weights, map_location=device) # load FP32 model imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 # if device.type != 'cpu' and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) # Half half = device.type != 'cpu' # half precision only supported on CUDA if half: model.half() # Configure model.eval() with open(data) as f: data = yaml.load(f, Loader=yaml.FullLoader) # model dict check_dataset(data) # check nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() # Dataloader if not training: img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img _ = model(img.half() if half else img) if device.type != 'cpu' else None # run once path = data['test'] if opt.task == 'test' else data['val'] # path to val/test images dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, hyp=None, augment=False, cache=False, pad=0.5, rect=True)[0] seen = 0 names = model.names if hasattr(model, 'names') else model.module.names coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', '[email protected]:.95') p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): img = img.to(device, non_blocking=True) img = img.half() if half else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width whwh = torch.Tensor([width, height, width, height]).to(device) # Disable gradients with torch.no_grad(): # Run model t = time_synchronized() inf_out, train_out = model(img, augment=augment) # inference and training outputs t0 += time_synchronized() - t # Compute loss if training: # if model has loss hyperparameters loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # GIoU, obj, cls # Run NMS t = time_synchronized() output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge) t1 += time_synchronized() - t # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh txt_path = str(out / Path(paths[si]).stem) pred[:, :4] = scale_coords(img[si].shape[1:], pred[:, :4], shapes[si][0], shapes[si][1]) # to original for *xyxy, conf, cls in pred: xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh with open(txt_path + '.txt', 'a') as f: f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format # Clip boxes to image bounds clip_coords(pred, (height, width)) # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = Path(paths[si]).stem box = pred[:, :4].clone() # xyxy scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({'image_id': int(image_id) if image_id.isnumeric() else image_id, 'category_id': coco91class[int(p[5])], 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5)}) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1) # best ious, indices # Append detections for j in (ious > iouv[0]).nonzero(as_tuple=False): d = ti[i[j]] # detected target if d not in detected: detected.append(d) correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn if len(detected) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Plot images if batch_i < 1: f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i) # filename plot_images(img, targets, paths, str(f), names) # ground truth f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i) plot_images(img, output_to_target(output, width, height), paths, str(f), names) # predictions # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats) and stats[0].any(): p, r, ap, f1, ap_class = ap_per_class(*stats) p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(1) # [P, R, [email protected], [email protected]:0.95] mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%12.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) # Print speeds t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple if not training: print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Save JSON if save_json and len(jdict): f = 'detections_val2017_%s_results.json' % \ (weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '') # filename print('\nCOCO mAP with pycocotools... saving %s...' % f) with open(f, 'w') as file: json.dump(jdict, file) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files] cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api cocoDt = cocoGt.loadRes(f) # initialize COCO pred api cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.params.imgIds = imgIds # image IDs to evaluate cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() map, map50 = cocoEval.stats[:2] # update results ([email protected]:0.95, [email protected]) except Exception as e: print('ERROR: pycocotools unable to run: %s' % e) # Return results model.float() # for training maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(val_dataset, model, decoder): results, image_ids = [], [] for index in range(len(val_dataset)): data = val_dataset[index] scale = data['scale'] cls_heads, reg_heads, batch_anchors = model(data['img'].cuda().permute( 2, 0, 1).float().unsqueeze(dim=0)) scores, classes, boxes = decoder(cls_heads, reg_heads, batch_anchors) scores, classes, boxes = scores.cpu(), classes.cpu(), boxes.cpu() boxes /= scale # make sure decode batch_size=1 # scores shape:[1,max_detection_num] # classes shape:[1,max_detection_num] # bboxes shape[1,max_detection_num,4] assert scores.shape[0] == 1 scores = scores.squeeze(0) classes = classes.squeeze(0) boxes = boxes.squeeze(0) # for coco_eval,we need [x_min,y_min,w,h] format pred boxes boxes[:, 2:] -= boxes[:, :2] for object_score, object_class, object_box in zip( scores, classes, boxes): object_score = float(object_score) object_class = int(object_class) object_box = object_box.tolist() if object_class == -1: break image_result = { 'image_id': val_dataset.image_ids[index], 'category_id': val_dataset.find_category_id_from_coco_label(object_class), 'score': object_score, 'bbox': object_box, } results.append(image_result) image_ids.append(val_dataset.image_ids[index]) print('{}/{}'.format(index, len(val_dataset)), end='\r') if not len(results): print("No target detected in test set images") return json.dump(results, open('{}_bbox_results.json'.format(val_dataset.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = val_dataset.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format( val_dataset.set_name)) coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() all_eval_result = coco_eval.stats return all_eval_result
# ''' # process annotations_GT # change box format from XYXY to XYWH (and adding area) for key, anno in cocoGt.anns.items(): [x1, y1, x2, y2] = anno['bbox'] x1 = min(x1, x2) x2 = max(x1, x2) y1 = min(y1, y2) y2 = max(y1, y2) width = abs(x2 - x1) height = abs(y2 - y1) # Don't do that # anno['bbox'] = [x1, y1, width, height] anno['area'] = width * height cocoGt.anns[key] = anno # start evaluation cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') # select N imgs to evaluate imgIds_sort = sorted(ImgIds) # cocoEval.params.imgIds = imgIds_sort[:500] cocoEval.evaluate() #评价 cocoEval.accumulate() #积累 cocoEval.summarize() #总结 exit()
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation limit: if not 0, it's the number of images to use for evaluation """ # Pick COCO images from the dataset image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] # Get corresponding COCO image IDs. coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): # Load image image = dataset.load_image(image_id) # Run detection t = time.time() r = model.detect([image], verbose=0)[0] t_prediction += (time.time() - t) # Convert results to COCO format # Cast masks to uint8 because COCO tools errors out on bool image_results = build_coco_results(dataset, coco_image_ids[i:i + 1], r["rois"], r["class_ids"], r["scores"], r["masks"].astype(np.uint8)) results.extend(image_results) # Load results. This modifies results with additional attributes. coco_results = coco.loadRes(results) # Evaluate cocoEval = COCOeval(coco, coco_results, eval_type) cocoEval.params.imgIds = coco_image_ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start) # # Subclass default COCO config for video inference (batch size > 1). # batch_size = 3 # class InferenceConfig(Config): # NAME="VideoInf" # GPU_COUNT = 1 # IMAGES_PER_GPU = batch_size # batch size. # DETECTION_MIN_CONFIDENCE = 0.6 # config = InferenceConfig() # config.display() # #@title Process video frame-by-frame # capture = cv2.VideoCapture("Test.mov") # print("Processing " + "Test.mov") # try: # if os.path.exists("Output"): # # Clear image cache. # shutil.rmtree("Output") # os.makedirs("Output") # except OSError: # pass # frames = [] # frame_count = 0 # while True: # ret, frame = capture.read() # if not ret: # break # # Buffer frames into batch to fit on GPU. # frame_count += 1 # frames.append(frame) # if len(frames) == batch_size: # results = model.detect(frames, verbose=0) # # Process footage frame-by-frame. # for i, (frame, r) in enumerate(zip(frames, results)): # # Filter results for Formalytics features. # frame = display_instances(frame, *filter_results(r)) # # Write processed frame back to image. # name = '{0}.jpg'.format(frame_count + i - batch_size) # name = os.path.join("Output", name) # cv2.imwrite(name, frame) # print('writing to file:{0}'.format(name)) # # Start next batch. # frames = [] # capture.release() # #@title Calculate video FPS # video = cv2.VideoCapture("Test.mov"); # # Find OpenCV version # (major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.') # if int(major_ver) < 3 : # fps = video.get(cv2.cv.CV_CAP_PROP_FPS) # print("Frames per second using video.get(cv2.cv.CV_CAP_PROP_FPS): {0}".format(fps)) # else : # fps = video.get(cv2.CAP_PROP_FPS) # print("Frames per second using video.get(cv2.CAP_PROP_FPS) : {0}".format(fps)) # video.release(); # #@title Write video # FPS = 60 #@param # def make_video(outvid, images=None, fps=FPS, size=None, # is_color=True, format="FMP4"): # """ # Create a video from a list of images. # @param outvid output video # @param images list of images to use in the video # @param fps frame per second # @param size size of each frame # @param is_color color # @param format see http://www.fourcc.org/codecs.php # @return see http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_gui/py_video_display/py_video_display.html # The function relies on http://opencv-python-tutroals.readthedocs.org/en/latest/. # By default, the video will have the size of the first image. # It will resize every image to this size before adding them to the video. # """ # from cv2 import VideoWriter, VideoWriter_fourcc, imread, resize # fourcc = VideoWriter_fourcc(*format) # vid = None # for image in images: # if not os.path.exists(image): # raise FileNotFoundError(image) # img = imread(image) # if vid is None: # if size is None: # size = img.shape[1], img.shape[0] # vid = VideoWriter(outvid, fourcc, float(fps), size, is_color) # if size[0] != img.shape[1] and size[1] != img.shape[0]: # img = resize(img, size) # vid.write(img) # vid.release() # return vid # import glob # import os # # Directory of images to run detection on # images = list(glob.iglob(os.path.join("Output", '*.*'))) # # Sort the images by integer index # images = sorted(images, key=lambda x: float(os.path.split(x)[1][:-3])) # outvid = os.path.join("./", "out.mp4") # make_video(outvid, images, fps=FPS) # !ls -alh ./videos/
from fast_rcnn.nms_wrapper import nms from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval import numpy as np import skimage.io as io import pylab if __name__ == '__main__': pylab.rcParams['figure.figsize'] = (10.0, 8.0) annType = 'bbox' ground_truth = '/mnt/d/BigData/COCO/instances_train-val2014/annotations/instances_val2014.json' generated_result = '/mnt/c/Users/Lavenger/git/py-faster-rcnn/tools/result.json' cocoGt = COCO(generated_result) cocoDt = cocoGt.loadRes(generated_result) cocoEval = COCOeval(cocoGt,cocoDt) cocoEval.params.imgIds = imgIds cocoEval.params.useSegm = False cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def eval_mscoco_with_segm(cocoGT, cocoPred): # running evaluation cocoEval = COCOeval(cocoGT, cocoPred, "keypoints") cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def main(): # pylint: disable=import-outside-toplevel from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval parser = make_parser() args = parser.parse_args() if args.end_epoch == -1: args.end_epoch = args.start_epoch for epoch_num in range(args.start_epoch, args.end_epoch + 1): if args.model: model_file = args.model else: model_file = "log-of-{}/epoch_{}.pkl".format( os.path.basename(args.file).split(".")[0], epoch_num) logger.info("Load Model : %s completed", model_file) results_list = list() result_queue = Queue(2000) procs = [] for i in range(args.ngpus): proc = Process( target=worker, args=( args.file, model_file, args.dataset_dir, i, args.ngpus, result_queue, ), ) proc.start() procs.append(proc) for _ in tqdm(range(5000)): results_list.append(result_queue.get()) for p in procs: p.join() sys.path.insert(0, os.path.dirname(args.file)) current_network = importlib.import_module( os.path.basename(args.file).split(".")[0]) cfg = current_network.Cfg() all_results = DetEvaluator.format(results_list, cfg) json_path = "log-of-{}/epoch_{}.json".format( os.path.basename(args.file).split(".")[0], epoch_num) all_results = json.dumps(all_results) with open(json_path, "w") as fo: fo.write(all_results) logger.info("Save to %s finished, start evaluation!", json_path) eval_gt = COCO( os.path.join(args.dataset_dir, cfg.test_dataset["name"], cfg.test_dataset["ann_file"])) eval_dt = eval_gt.loadRes(json_path) cocoEval = COCOeval(eval_gt, eval_dt, iouType="bbox") cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() metrics = [ "AP", "[email protected]", "[email protected]", "APs", "APm", "APl", "AR@1", "AR@10", "AR@100", "ARs", "ARm", "ARl", ] logger.info("mmAP".center(32, "-")) for i, m in enumerate(metrics): logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i]) logger.info("-" * 32)
def main(): args = get_args() cocoGt = COCO( os.path.join(args.coco_dir, "annotations/instances_val2017.json")) if args.use_inv_map: inv_map = [0 ] + cocoGt.getCatIds() # First label in inv_map is not used with open(args.mlperf_accuracy_file, "r") as f: results = json.load(f) detections = [] image_ids = set() seen = set() no_results = 0 if args.remove_48_empty_images: im_ids = [] for i in cocoGt.getCatIds(): im_ids += cocoGt.catToImgs[i] im_ids = list(set(im_ids)) image_map = [cocoGt.imgs[id] for id in im_ids] else: image_map = cocoGt.dataset["images"] for j in results: idx = j['qsl_idx'] # de-dupe in case loadgen sends the same image multiple times if idx in seen: continue seen.add(idx) # reconstruct from mlperf accuracy log # what is written by the benchmark is an array of float32's: # id, box[0], box[1], box[2], box[3], score, detection_class # note that id is a index into instances_val2017.json, not the actual image_id data = np.frombuffer(bytes.fromhex(j['data']), np.float32) if len(data) < 7: # handle images that had no results image = image_map[idx] # by adding the id to image_ids we make pycoco aware of the no-result image image_ids.add(image["id"]) no_results += 1 if args.verbose: print("no results: {}, idx={}".format(image["coco_url"], idx)) continue for i in range(0, len(data), 7): image_idx, ymin, xmin, ymax, xmax, score, label = data[i:i + 7] image = image_map[idx] image_idx = int(image_idx) if image_idx != idx: print( "ERROR: loadgen({}) and payload({}) disagree on image_idx". format(idx, image_idx)) image_id = image["id"] height, width = image["height"], image["width"] ymin *= height xmin *= width ymax *= height xmax *= width loc = os.path.join(args.coco_dir, "val2017", image["file_name"]) label = int(label) if args.use_inv_map: label = inv_map[label] # pycoco wants {imageID,x1,y1,w,h,score,class} detections.append({ "image_id": image_id, "image_loc": loc, "category_id": label, "bbox": [ float(xmin), float(ymin), float(xmax - xmin), float(ymax - ymin) ], "score": float(score) }) image_ids.add(image_id) with open(args.output_file, "w") as fp: json.dump(detections, fp, sort_keys=True, indent=4) cocoDt = cocoGt.loadRes( args.output_file) # Load from file to bypass error with Python3 cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox') cocoEval.params.imgIds = list(image_ids) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("mAP={:.3f}%".format(100. * cocoEval.stats[0])) if args.verbose: print("found {} results".format(len(results))) print("found {} images".format(len(image_ids))) print("found {} images with no results".format(no_results)) print("ignored {} dupes".format(len(results) - len(seen)))
def evaluate_mAP( res_file, ann_type='bbox', ann_file='./data/coco/annotations/person_keypoints_val2017.json', silence=True): """Evaluate mAP result for coco dataset. Parameters ---------- res_file: str Path to result json file. ann_type: str annotation type, including: `bbox`, `segm`, `keypoints`. ann_file: str Path to groundtruth file. silence: bool True: disable running log. """ class NullWriter(object): def write(self, arg): pass # ann_file = os.path.join('./data/coco/annotations/', ann_file) if silence: nullwrite = NullWriter() oldstdout = sys.stdout sys.stdout = nullwrite # disable output cocoGt = COCO(ann_file) cocoDt = cocoGt.loadRes(res_file) cocoEval = COCOeval(cocoGt, cocoDt, ann_type) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() if silence: sys.stdout = oldstdout # enable output if isinstance(cocoEval.stats[0], dict): stats_names = [ 'AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] parts = ['body', 'face', 'hand', 'fullbody'] info = {} for i, part in enumerate(parts): info[part] = cocoEval.stats[i][part][0] return info else: stats_names = [ 'AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] info_str = {} for ind, name in enumerate(stats_names): info_str[name] = cocoEval.stats[ind] return info_str['AP']
def benchmark_model(frozen_graph, images_dir, annotation_path, batch_size=1, image_shape=None, num_images=4096, tmp_dir='.benchmark_model_tmp_dir', remove_tmp_dir=True, output_path=None, display_every=100): """Computes accuracy and performance statistics Computes accuracy and performance statistics by executing over many images from the MSCOCO dataset defined by images_dir and annotation_path. Args ---- frozen_graph: A GraphDef representing the object detection model to test. Alternatively, a string representing the path to the saved frozen graph. images_dir: A string representing the path of the COCO images directory. annotation_path: A string representing the path of the COCO annotation file. batch_size: An integer representing the batch size to use when feeding images to the model. image_shape: An optional tuple of integers representing a fixed shape to resize all images before testing. num_images: An integer representing the number of images in the dataset to evaluate with. tmp_dir: A string representing the path where the function may create a temporary directory to store intermediate files. output_path: An optional string representing a path to store the statistics in JSON format. display_every: int, print log every display_every iteration Returns ------- statistics: A named dictionary of accuracy and performance statistics computed for the model. """ if os.path.exists(tmp_dir): if not remove_tmp_dir: raise RuntimeError('Temporary directory exists; %s' % tmp_dir) subprocess.call(['rm', '-rf', tmp_dir]) if batch_size > 1 and image_shape is None: raise RuntimeError( 'Fixed image shape must be provided for batch size > 1') from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval coco = COCO(annotation_file=annotation_path) # get list of image ids to use for evaluation image_ids = coco.getImgIds() if num_images > len(image_ids): print( 'Num images provided %d exceeds number in dataset %d, using %d images instead' % (num_images, len(image_ids), len(image_ids))) num_images = len(image_ids) image_ids = image_ids[0:num_images] # load frozen graph from file if string, otherwise must be GraphDef if isinstance(frozen_graph, str): frozen_graph_path = frozen_graph frozen_graph = tf.GraphDef() with open(frozen_graph_path, 'rb') as f: frozen_graph.ParseFromString(f.read()) elif not isinstance(frozen_graph, tf.GraphDef): raise TypeError('Expected frozen_graph to be GraphDef or str') tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True coco_detections = [] # list of all bounding box detections in coco format runtimes = [] # list of runtimes for each batch image_counts = [] # list of number of images in each batch with tf.Graph().as_default() as tf_graph: with tf.Session(config=tf_config) as tf_sess: tf.import_graph_def(frozen_graph, name='') tf_input = tf_graph.get_tensor_by_name(INPUT_NAME + ':0') tf_boxes = tf_graph.get_tensor_by_name(BOXES_NAME + ':0') tf_classes = tf_graph.get_tensor_by_name(CLASSES_NAME + ':0') tf_scores = tf_graph.get_tensor_by_name(SCORES_NAME + ':0') tf_num_detections = tf_graph.get_tensor_by_name( NUM_DETECTIONS_NAME + ':0') # load batches from coco dataset for image_idx in range(0, len(image_ids), batch_size): batch_image_ids = image_ids[image_idx:image_idx + batch_size] batch_images = [] batch_coco_images = [] # read images from file for image_id in batch_image_ids: coco_img = coco.imgs[image_id] batch_coco_images.append(coco_img) image_path = os.path.join(images_dir, coco_img['file_name']) image = _read_image(image_path, image_shape) batch_images.append(image) # run once outside of timing to initialize if image_idx == 0: boxes, classes, scores, num_detections = tf_sess.run( [tf_boxes, tf_classes, tf_scores, tf_num_detections], feed_dict={tf_input: batch_images}) # execute model and compute time difference t0 = time.time() boxes, classes, scores, num_detections = tf_sess.run( [tf_boxes, tf_classes, tf_scores, tf_num_detections], feed_dict={tf_input: batch_images}) t1 = time.time() # log runtime and image count runtimes.append(float(t1 - t0)) if len(runtimes) % display_every == 0: print(" step %d/%d, iter_time(ms)=%.4f" % (len(runtimes), (len(image_ids) + batch_size - 1) / batch_size, np.mean(runtimes) * 1000)) image_counts.append(len(batch_images)) # add coco detections for this batch to running list batch_coco_detections = [] for i, image_id in enumerate(batch_image_ids): image_width = batch_coco_images[i]['width'] image_height = batch_coco_images[i]['height'] for j in range(int(num_detections[i])): bbox = boxes[i][j] bbox_coco_fmt = [ bbox[1] * image_width, # x0 bbox[0] * image_height, # x1 (bbox[3] - bbox[1]) * image_width, # width (bbox[2] - bbox[0]) * image_height, # height ] coco_detection = { 'image_id': image_id, 'category_id': int(classes[i][j]), 'bbox': bbox_coco_fmt, 'score': float(scores[i][j]) } coco_detections.append(coco_detection) # write coco detections to file subprocess.call(['mkdir', '-p', tmp_dir]) coco_detections_path = os.path.join(tmp_dir, 'coco_detections.json') with open(coco_detections_path, 'w') as f: json.dump(coco_detections, f) # compute coco metrics cocoDt = coco.loadRes(coco_detections_path) eval = COCOeval(coco, cocoDt, 'bbox') eval.params.imgIds = image_ids eval.evaluate() eval.accumulate() eval.summarize() statistics = { 'map': eval.stats[0], 'avg_latency_ms': 1000.0 * np.mean(runtimes), 'avg_throughput_fps': np.sum(image_counts) / np.sum(runtimes), 'runtimes_ms': [1000.0 * r for r in runtimes] } if output_path is not None: subprocess.call(['mkdir', '-p', os.path.dirname(output_path)]) with open(output_path, 'w') as f: json.dump(statistics, f) subprocess.call(['rm', '-rf', tmp_dir]) return statistics
def main(): # pylint: disable=import-outside-toplevel,too-many-branches,too-many-statements from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval parser = make_parser() args = parser.parse_args() current_network = import_from_file(args.file) cfg = current_network.Cfg() if args.weight_file: args.start_epoch = args.end_epoch = -1 else: if args.start_epoch == -1: args.start_epoch = cfg.max_epoch - 1 if args.end_epoch == -1: args.end_epoch = args.start_epoch assert 0 <= args.start_epoch <= args.end_epoch < cfg.max_epoch for epoch_num in range(args.start_epoch, args.end_epoch + 1): if args.weight_file: weight_file = args.weight_file else: weight_file = "log-of-{}/epoch_{}.pkl".format( os.path.basename(args.file).split(".")[0], epoch_num) if args.ngpus > 1: master_ip = "localhost" port = dist.get_free_ports(1)[0] dist.Server(port) result_list = [] result_queue = Queue(2000) procs = [] for i in range(args.ngpus): proc = Process( target=worker, args=( current_network, weight_file, args.dataset_dir, master_ip, port, args.ngpus, i, result_queue, ), ) proc.start() procs.append(proc) num_imgs = dict(coco=5000, cocomini=5000, objects365=30000) for _ in tqdm(range(num_imgs[cfg.test_dataset["name"]])): result_list.append(result_queue.get()) for p in procs: p.join() else: result_list = [] worker(current_network, weight_file, args.dataset_dir, None, None, 1, 0, result_list) total_time = sum([x["perf_time"] for x in result_list]) average_time = total_time / len(result_list) fps = 1.0 / average_time logger.info("average inference speed: {:.4}s / iter, fps:{:.3}".format( average_time, fps)) all_results = DetEvaluator.format(result_list, cfg) json_path = "log-of-{}/epoch_{}.json".format( os.path.basename(args.file).split(".")[0], epoch_num) all_results = json.dumps(all_results) with open(json_path, "w") as fo: fo.write(all_results) logger.info("Save to %s finished, start evaluation!", json_path) eval_gt = COCO( os.path.join(args.dataset_dir, cfg.test_dataset["name"], cfg.test_dataset["ann_file"])) eval_dt = eval_gt.loadRes(json_path) cocoEval = COCOeval(eval_gt, eval_dt, iouType="bbox") cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() metrics = [ "AP", "[email protected]", "[email protected]", "APs", "APm", "APl", "AR@1", "AR@10", "AR@100", "ARs", "ARm", "ARl", ] logger.info("mmAP".center(32, "-")) for i, m in enumerate(metrics): logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i]) logger.info("-" * 32)
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation limit: if not 0, it's the number of images to use for evaluation """ # Pick COCO images from the dataset image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] # Get corresponding COCO image IDs. coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): # Load image image = dataset.load_image(image_id) # Run detection t = time.time() r = model.detect([image])[0] t_prediction += (time.time() - t) # Convert results to COCO format image_results = build_coco_results(dataset, coco_image_ids[i:i + 1], r["rois"], r["class_ids"], r["scores"], r["masks"]) results.extend(image_results) # Load results. This modifies results with additional attributes. coco_results = coco.loadRes(results) # Evaluate cocoEval = COCOeval(coco, coco_results, eval_type) cocoEval.params.imgIds = coco_image_ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start)
def evaluate_coco(dataset, model, threshold=0.05): model.eval() device = [p.device for p in model.parameters()][0] with torch.no_grad(): # start collecting results results = [] image_ids = [] for index in range(len(dataset)): data = dataset[index] scale = data['scale'] # run network #TODO: question, why is this permuted bellow? scores, labels, boxes = model( data.image.permute( 2, 0, 1).to(device=device).float().unsqueeze(dim=0)) scores = scores.cpu() labels = labels.cpu() boxes = boxes.cpu() # correct boxes for image scale boxes /= scale if boxes.shape[0] > 0: # change to (x, y, w, h) (MS COCO standard) boxes[:, 2] -= boxes[:, 0] boxes[:, 3] -= boxes[:, 1] # compute predicted labels and scores #for box, score, label in zip(boxes[0], scores[0], labels[0]): for box_id in range(boxes.shape[0]): score = float(scores[box_id]) label = int(labels[box_id]) box = boxes[box_id, :] # scores are sorted, so we can break if score < threshold: break # append detection for each positively labeled class image_result = { 'image_id': dataset.image_ids[index], 'category_id': dataset.label_to_coco_label(label), 'score': float(score), 'bbox': box.tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(dataset.image_ids[index]) # print progress print('{}/{}'.format(index, len(dataset)), end='\r') if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(dataset.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = dataset.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format( dataset.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() model.train() return