def evaluate():
cocoGt = COCO('annotations.json')
cocoDt = cocoGt.loadRes('detections.json')
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def _do_python_eval(self, _coco):
coco_dt = _coco.loadRes(self._result_file)
coco_eval = COCOeval(_coco, coco_dt)
coco_eval.params.useSegm = False
coco_eval.evaluate()
coco_eval.accumulate()
self._print_detection_metrics(coco_eval)
def coco_evaluate(json_dataset, res_file, image_ids):
coco_dt = json_dataset.COCO.loadRes(str(res_file))
coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval
def _do_segmentation_eval(json_dataset, res_file, output_dir):
coco_dt = json_dataset.COCO.loadRes(str(res_file))
coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'segm')
coco_eval.evaluate()
coco_eval.accumulate()
_log_detection_eval_metrics(json_dataset, coco_eval)
eval_file = os.path.join(output_dir, 'segmentation_results.pkl')
robust_pickle_dump(coco_eval, eval_file)
logger.info('Wrote json eval results to: {}'.format(eval_file))
def _do_detection_eval(json_dataset, res_file, output_dir):
coco_dt = json_dataset.COCO.loadRes(str(res_file))
coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox')
coco_eval.evaluate()
coco_eval.accumulate()
_log_detection_eval_metrics(json_dataset, coco_eval)
eval_file = os.path.join(output_dir, 'detection_results.pkl')
save_object(coco_eval, eval_file)
logger.info('Wrote json eval results to: {}'.format(eval_file))
return coco_eval
def cocoval(detected_json):
eval_json = config.eval_json
eval_gt = COCO(eval_json)
eval_dt = eval_gt.loadRes(detected_json)
cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox')
# cocoEval.params.imgIds = eval_gt.getImgIds()
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def _do_coco_eval(self, dtFile, output_dir):
"""
Evaluate using COCO API
"""
if self._image_set == 'train' or self._image_set == 'val':
cocoGt = self._coco[0]
cocoDt = COCO(dtFile)
E = COCOeval(cocoGt, cocoDt)
E.evaluate()
E.accumulate()
E.summarize()
def evaluate_detections(self, all_boxes, output_dir=None):
resFile = self._write_coco_results_file(all_boxes)
cocoGt = self._annotations
cocoDt = cocoGt.loadRes(resFile)
# running evaluation
cocoEval = COCOeval(cocoGt,cocoDt)
# useSegm should default to 0
#cocoEval.params.useSegm = 0
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def compute_ap(self):
coco_res = self.loader.coco.loadRes(self.filename)
cocoEval = COCOeval(self.loader.coco, coco_res)
cocoEval.params.imgIds = self.loader.get_filenames()
cocoEval.params.useSegm = False
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
return cocoEval
def _do_detection_eval(self, res_file, output_dir):
ann_type = 'bbox'
coco_dt = self._COCO.loadRes(res_file)
coco_eval = COCOeval(self._COCO, coco_dt)
coco_eval.params.useSegm = (ann_type == 'segm')
coco_eval.evaluate()
coco_eval.accumulate()
self._print_detection_eval_metrics(coco_eval)
eval_file = osp.join(output_dir, 'detection_results.pkl')
with open(eval_file, 'wb') as fid:
pickle.dump(coco_eval, fid, pickle.HIGHEST_PROTOCOL)
print('Wrote COCO eval results to: {}'.format(eval_file))
def _do_keypoint_eval(json_dataset, res_file, output_dir):
ann_type = 'keypoints'
imgIds = json_dataset.COCO.getImgIds()
imgIds.sort()
coco_dt = json_dataset.COCO.loadRes(res_file)
coco_eval = COCOeval(json_dataset.COCO, coco_dt, ann_type)
coco_eval.params.imgIds = imgIds
coco_eval.evaluate()
coco_eval.accumulate()
eval_file = os.path.join(output_dir, 'keypoint_results.pkl')
robust_pickle_dump(coco_eval, eval_file)
logger.info('Wrote json eval results to: {}'.format(eval_file))
coco_eval.summarize()
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
# Pick COCO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
# Get corresponding COCO image IDs.
coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
# Load image
image = dataset.load_image(image_id)
# Run detection
t = time.time()
r = model.detect([image], verbose=0)[0]
t_prediction += (time.time() - t)
# Convert results to COCO format
# Cast masks to uint8 because COCO tools errors out on bool
image_results = build_coco_results(dataset, coco_image_ids[i:i + 1],
r["rois"], r["class_ids"],
r["scores"],
r["masks"].astype(np.uint8))
results.extend(image_results)
# Load results. This modifies results with additional attributes.
coco_results = coco.loadRes(results)
# Evaluate
cocoEval = COCOeval(coco, coco_results, eval_type)
cocoEval.params.imgIds = coco_image_ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
def validate(val_loader, model, i, silence=False):
batch_time = AverageMeter()
coco_gt = val_loader.dataset.coco
coco_pred = COCO()
coco_pred.dataset['images'] = [img for img in coco_gt.datasets['images']]
coco_pred.dataset['categories'] = copy.deepcopy(coco_gt.dataset['categories'])
id = 0
# switch to evaluate mode
model.eval()
end = time.time()
for i, (inputs, anns) in enumerate(val_loader):
# forward images one by one (TODO: support batch mode later, or
# multiprocess)
for j, input in enumerate(inputs):
input_anns= anns[j] # anns of this input
gt_bbox= np.vstack([ann['bbox'] + [ann['ordered_id']] for ann in input_anns])
im_info= [[input.size(1), input.size(2),
input_anns[0]['scale_ratio']]]
input_var= Variable(input.unsqueeze(0),
requires_grad=False).cuda()
cls_prob, bbox_pred, rois = model(input_var, im_info)
scores, pred_boxes = model.interpret_outputs(cls_prob, bbox_pred, rois, im_info)
print(scores, pred_boxes)
# for i in range(scores.shape[0]):
# measure elapsed time
batch_time.update(time.time() - end)
end= time.time()
coco_pred.createIndex()
coco_eval = COCOeval(coco_gt, coco_pred, 'bbox')
coco_eval.params.imgIds= sorted(coco_gt.getImgIds())
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
print('iter: [{0}] '
'Time {batch_time.avg:.3f} '
'Val Stats: {1}'
.format(i, coco_eval.stats,
batch_time=batch_time))
return coco_eval.stats[0]
def _do_eval(res_file, output_dir,_COCO,classes):
## The function is borrowed from https://github.com/rbgirshick/fast-rcnn/ and changed
ann_type = 'bbox'
coco_dt = _COCO.loadRes(res_file)
coco_eval = COCOeval(_COCO, coco_dt)
coco_eval.params.useSegm = (ann_type == 'segm')
coco_eval.evaluate()
coco_eval.accumulate()
_print_eval_metrics(coco_eval,classes)
# Write the result file
eval_file = osp.join(output_dir)
eval_result = {}
eval_result['precision'] = coco_eval.eval['precision']
eval_result['recall'] = coco_eval.eval['recall']
sio.savemat(eval_file,eval_result)
print 'Wrote COCO eval results to: {}'.format(eval_file)
def evaluate_predictions_on_coco(
coco_gt, coco_results, json_result_file, iou_type="bbox"
):
import json
with open(json_result_file, "w") as f:
json.dump(coco_results, f)
from pycocotools.cocoeval import COCOeval
coco_dt = coco_gt.loadRes(str(json_result_file))
# coco_dt = coco_gt.loadRes(coco_results)
coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval
def calc_coco_metrics(coco_annotations, predictions, classes):
annotations = ObjectDetectorJson.convert_coco_to_toolbox_format(coco_annotations, classes)
detections = []
for annotation, prediction in zip(annotations, predictions):
width, height = annotation['image_size']
image_id = annotation['image_id']
for obj_id, obj in enumerate(prediction):
label = int(obj[1])
score = float(obj[2])
if obj_id != 0 and score == 0: # At least one prediction must be (COCO API issue)
continue
bbox = (obj[3:]).tolist()
bbox[::2] = [width * i for i in bbox[::2]]
bbox[1::2] = [height * i for i in bbox[1::2]]
xmin, ymin, xmax, ymax = bbox
w_bbox = round(xmax - xmin, 1)
h_bbox = round(ymax - ymin, 1)
xmin, ymin = round(xmin, 1), round(ymin, 1)
coco_det = {}
coco_det['image_id'] = image_id
coco_det['category_id'] = label
coco_det['bbox'] = [xmin, ymin, w_bbox, h_bbox]
coco_det['score'] = score
detections.append(coco_det)
coco_dt = coco_annotations.loadRes(detections)
img_ids = sorted(coco_annotations.getImgIds())
coco_eval = COCOeval(coco_annotations, coco_dt, 'bbox')
coco_eval.params.imgIds = img_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
metrics = {}
for metric_name, value in zip(METRICS_NAMES, coco_eval.stats):
metrics[metric_name] = value
return metrics
def print_evaluation_scores(json_file):
ret = {}
assert config.BASEDIR and os.path.isdir(config.BASEDIR)
annofile = os.path.join(
config.BASEDIR, 'annotations',
'instances_{}.json'.format(config.VAL_DATASET))
coco = COCO(annofile)
cocoDt = coco.loadRes(json_file)
cocoEval = COCOeval(coco, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
ret['mAP(bbox)'] = cocoEval.stats[0]
if config.MODE_MASK:
cocoEval = COCOeval(coco, cocoDt, 'segm')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
ret['mAP(segm)'] = cocoEval.stats[0]
return ret
def _update(self):
"""Use coco to get real scores. """
if not self._current_id == len(self._img_ids):
warnings.warn(
'Recorded {} out of {} validation images, incompelete results'.format(
self._current_id, len(self._img_ids)))
import json
try:
with open(self._filename, 'w') as f:
json.dump(self._results, f)
except IOError as e:
raise RuntimeError("Unable to dump json file, ignored. What(): {}".format(str(e)))
pred = self.dataset.coco.loadRes(self._filename)
gt = self.dataset.coco
# lazy import pycocotools
try_import_pycocotools()
from pycocotools.cocoeval import COCOeval
coco_eval = COCOeval(gt, pred, 'bbox')
coco_eval.evaluate()
coco_eval.accumulate()
self._coco_eval = coco_eval
return coco_eval
def evaluate_coco(model, dataset, coco, config, eval_type="bbox", limit=None, image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
"""
# Pick COCO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
# Get corresponding COCO image IDs.
coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
if i%10==0:
print('Processed %d images'%i )
# Load image
image = dataset.load_image(image_id)
# Run detection
t = time.time()
r = inference(image, model, config)
t_prediction += (time.time() - t)
# Convert results to COCO format
image_results = build_coco_results(dataset, coco_image_ids[i:i + 1],
r["rois"], r["class_ids"],
r["scores"], r["masks"])
results.extend(image_results)
# Load results. This modifies results with additional attributes.
coco_results = coco.loadRes(results)
# Evaluate
cocoEval = COCOeval(coco, coco_results, eval_type)
cocoEval.params.imgIds = coco_image_ids
# Only evaluate for person.
cocoEval.params.catIds = coco.getCatIds(catNms=['person'])
cocoEval.evaluate()
a=cocoEval.accumulate()
b=cocoEval.summarize()
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
def print_evaluation_scores(json_file):
ret = {}
assert config.BASEDIR and os.path.isdir(config.BASEDIR)
annofile = os.path.join(
config.BASEDIR, 'annotations',
'instances_{}.json'.format(config.VAL_DATASET))
coco = COCO(annofile)
cocoDt = coco.loadRes(json_file)
cocoEval = COCOeval(coco, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
fields = ['IoU=0.5:0.95', 'IoU=0.5', 'IoU=0.75', 'small', 'medium', 'large']
for k in range(6):
ret['mAP(bbox)/' + fields[k]] = cocoEval.stats[k]
if config.MODE_MASK:
cocoEval = COCOeval(coco, cocoDt, 'segm')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
for k in range(6):
ret['mAP(segm)/' + fields[k]] = cocoEval.stats[k]
return ret
def main():
# pylint: disable=import-outside-toplevel
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
parser = make_parser()
args = parser.parse_args()
sys.path.insert(0, os.path.dirname(args.file))
current_network = importlib.import_module(
os.path.basename(args.file).split(".")[0])
cfg = current_network.Cfg()
if args.weight_file:
args.start_epoch = args.end_epoch = -1
else:
if args.start_epoch == -1:
args.start_epoch = cfg.max_epoch - 1
if args.end_epoch == -1:
args.end_epoch = args.start_epoch
assert 0 <= args.start_epoch <= args.end_epoch < cfg.max_epoch
for epoch_num in range(args.start_epoch, args.end_epoch + 1):
if args.weight_file:
model_file = args.weight_file
else:
model_file = "log-of-{}/epoch_{}.pkl".format(
os.path.basename(args.file).split(".")[0], epoch_num)
logger.info("Load Model : %s completed", model_file)
results_list = list()
result_queue = Queue(2000)
procs = []
for i in range(args.ngpus):
proc = Process(
target=worker,
args=(
current_network,
model_file,
args.dataset_dir,
i,
args.ngpus,
result_queue,
),
)
proc.start()
procs.append(proc)
num_imgs = dict(coco=5000, objects365=30000)
for _ in tqdm(range(num_imgs[cfg.test_dataset["name"]])):
results_list.append(result_queue.get())
for p in procs:
p.join()
all_results = DetEvaluator.format(results_list, cfg)
json_path = "log-of-{}/epoch_{}.json".format(
os.path.basename(args.file).split(".")[0], epoch_num)
all_results = json.dumps(all_results)
with open(json_path, "w") as fo:
fo.write(all_results)
logger.info("Save to %s finished, start evaluation!", json_path)
eval_gt = COCO(
os.path.join(args.dataset_dir, cfg.test_dataset["name"],
cfg.test_dataset["ann_file"]))
eval_dt = eval_gt.loadRes(json_path)
cocoEval = COCOeval(eval_gt, eval_dt, iouType="bbox")
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
metrics = [
"AP",
"[email protected]",
"[email protected]",
"APs",
"APm",
"APl",
"AR@1",
"AR@10",
"AR@100",
"ARs",
"ARm",
"ARl",
]
logger.info("mmAP".center(32, "-"))
for i, m in enumerate(metrics):
logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i])
logger.info("-" * 32)
def main():
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
parser = make_parser()
args = parser.parse_args()
model_name = "{}_{}x{}".format(args.arch, cfg.input_shape[0],
cfg.input_shape[1])
save_dir = os.path.join(args.save_dir, model_name)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
mge.set_log_file(os.path.join(save_dir, "log.txt"))
args.ngpus = (dist.helper.get_device_count_by_fork("gpu")
if args.ngpus is None else args.ngpus)
cfg.batch_size = cfg.batch_size if args.batch_size is None else args.batch_size
dt_path = os.path.join(cfg.data_root, "person_detection_results",
args.dt_file)
dets = json.load(open(dt_path, "r"))
gt_path = os.path.join(cfg.data_root, "annotations",
"person_keypoints_val2017.json")
eval_gt = COCO(gt_path)
gt = eval_gt.dataset
dets = [
i for i in dets
if (i["image_id"] in eval_gt.imgs and i["category_id"] == 1)
]
ann_file = {"images": gt["images"], "annotations": dets}
if args.end_epoch == -1:
args.end_epoch = args.start_epoch
for epoch_num in range(args.start_epoch, args.end_epoch + 1,
args.test_freq):
if args.model:
model_file = args.model
else:
model_file = "{}/epoch_{}.pkl".format(args.model_dir, epoch_num)
logger.info("Load Model : %s completed", model_file)
dist_worker = dist.launcher(n_gpus=args.ngpus)(worker)
all_results = dist_worker(args.arch, model_file, cfg.data_root,
ann_file)
all_results = sum(all_results, [])
json_name = "log-of-{}_epoch_{}.json".format(args.arch, epoch_num)
json_path = os.path.join(save_dir, json_name)
all_results = json.dumps(all_results)
with open(json_path, "w") as fo:
fo.write(all_results)
logger.info("Save to %s finished, start evaluation!", json_path)
eval_dt = eval_gt.loadRes(json_path)
cocoEval = COCOeval(eval_gt, eval_dt, iouType="keypoints")
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
metrics = [
"AP",
"[email protected]",
"[email protected]",
"APm",
"APl",
"AR",
"[email protected]",
"[email protected]",
"ARm",
"ARl",
]
logger.info("mmAP".center(32, "-"))
for i, m in enumerate(metrics):
logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i])
logger.info("-" * 32)
def run(
data,
weights=None, # model.pt path(s)
batch_size=32, # batch size
imgsz=640, # inference size (pixels)
conf_thres=0.001, # confidence threshold
iou_thres=0.6, # NMS IoU threshold
task='val', # train, val, test, speed or study
device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
single_cls=False, # treat as single-class dataset
augment=False, # augmented inference
verbose=False, # verbose output
save_txt=False, # save results to *.txt
save_hybrid=False, # save label+prediction hybrid results to *.txt
save_conf=False, # save confidences in --save-txt labels
save_json=False, # save a COCO-JSON results file
project=ROOT / 'runs/val', # save to project/name
name='exp', # save to project/name
exist_ok=False, # existing project/name ok, do not increment
half=True, # use FP16 half-precision inference
model=None,
dataloader=None,
save_dir=Path(''),
plots=True,
callbacks=Callbacks(),
compute_loss=None,
):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
device = select_device(device, batch_size=batch_size)
# Directories
save_dir = increment_path(Path(project) / name,
exist_ok=exist_ok) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(
parents=True, exist_ok=True) # make dir
# Load model
check_suffix(weights, '.pt')
model = attempt_load(weights, map_location=device) # load FP32 model
gs = max(int(model.stride.max()), 32) # grid size (max stride)
imgsz = check_img_size(imgsz, s=gs) # check image size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Data
data = check_dataset(data) # check
# Half
half &= device.type != 'cpu' # half precision only supported on CUDA
model.half() if half else model.float()
# Configure
model.eval()
is_coco = isinstance(data.get('val'), str) and data['val'].endswith(
'coco/val2017.txt') # COCO dataset
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if not training:
if device.type != 'cpu':
model(
torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(
next(model.parameters()))) # run once
pad = 0.0 if task == 'speed' else 0.5
task = task if task in (
'train', 'val', 'test') else 'val' # path to train/val/test images
dataloader = create_dataloader(data[task],
imgsz,
batch_size,
gs,
single_cls,
pad=pad,
rect=True,
prefix=colorstr(f'{task}: '))[0]
seen = 0
confusion_matrix = ConfusionMatrix(nc=nc)
names = {
k: v
for k, v in enumerate(
model.names if hasattr(model, 'names') else model.module.names)
}
class_map = coco80_to_coco91_class() if is_coco else list(range(1000))
s = ('%20s' + '%11s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R',
'[email protected]', '[email protected]:.95')
dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0,
0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (img, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=s)):
t1 = time_sync()
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
t2 = time_sync()
dt[0] += t2 - t1
# Run model
out, train_out = model(
img, augment=augment) # inference and training outputs
dt[1] += time_sync() - t2
# Compute loss
if compute_loss:
loss += compute_loss([x.float() for x in train_out],
targets)[1] # box, obj, cls
# Run NMS
targets[:, 2:] *= torch.Tensor([width, height, width,
height]).to(device) # to pixels
lb = [targets[targets[:, 0] == i, 1:]
for i in range(nb)] if save_hybrid else [] # for autolabelling
t3 = time_sync()
out = non_max_suppression(out,
conf_thres,
iou_thres,
labels=lb,
multi_label=True,
agnostic=single_cls)
dt[2] += time_sync() - t3
# Statistics per image
for si, pred in enumerate(out):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
path, shape = Path(paths[si]), shapes[si][0]
seen += 1
if len(pred) == 0:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Predictions
if single_cls:
pred[:, 5] = 0
predn = pred.clone()
scale_coords(img[si].shape[1:], predn[:, :4], shape,
shapes[si][1]) # native-space pred
# Evaluate
if nl:
tbox = xywh2xyxy(labels[:, 1:5]) # target boxes
scale_coords(img[si].shape[1:], tbox, shape,
shapes[si][1]) # native-space labels
labelsn = torch.cat((labels[:, 0:1], tbox),
1) # native-space labels
correct = process_batch(predn, labelsn, iouv)
if plots:
confusion_matrix.process_batch(predn, labelsn)
else:
correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool)
stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(),
tcls)) # (correct, conf, pcls, tcls)
# Save/log
if save_txt:
save_one_txt(predn,
save_conf,
shape,
file=save_dir / 'labels' / (path.stem + '.txt'))
if save_json:
save_one_json(predn, jdict, path,
class_map) # append to COCO-JSON dictionary
callbacks.run('on_val_image_end', pred, predn, path, names,
img[si])
# Plot images
if plots and batch_i < 3:
f = save_dir / f'val_batch{batch_i}_labels.jpg' # labels
Thread(target=plot_images,
args=(img, targets, paths, f, names),
daemon=True).start()
f = save_dir / f'val_batch{batch_i}_pred.jpg' # predictions
Thread(target=plot_images,
args=(img, output_to_target(out), paths, f, names),
daemon=True).start()
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats,
plot=plots,
save_dir=save_dir,
names=names)
ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%11i' * 2 + '%11.3g' * 4 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3 for x in dt) # speeds per image
if not training:
shape = (batch_size, 3, imgsz, imgsz)
print(
f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}'
% t)
# Plots
if plots:
confusion_matrix.plot(save_dir=save_dir, names=list(names.values()))
callbacks.run('on_val_end')
# Save JSON
if save_json and len(jdict):
w = Path(weights[0] if isinstance(weights, list) else weights
).stem if weights is not None else '' # weights
anno_json = str(
Path(data.get('path', '../coco')) /
'annotations/instances_val2017.json') # annotations json
pred_json = str(save_dir / f"{w}_predictions.json") # predictions json
print(f'\nEvaluating pycocotools mAP... saving {pred_json}...')
with open(pred_json, 'w') as f:
json.dump(jdict, f)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
check_requirements(['pycocotools'])
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
anno = COCO(anno_json) # init annotations api
pred = anno.loadRes(pred_json) # init predictions api
eval = COCOeval(anno, pred, 'bbox')
if is_coco:
eval.params.imgIds = [
int(Path(x).stem) for x in dataloader.dataset.img_files
] # image IDs to evaluate
eval.evaluate()
eval.accumulate()
eval.summarize()
map, map50 = eval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print(f'pycocotools unable to run: {e}')
# Return results
model.float() # for training
if not training:
s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
print(f"Results saved to {colorstr('bold', save_dir)}{s}")
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(coco_gt, coco_dt, parameters, catIds=None):
LOG = getLogger('processor.EvaluateSegmentation')
LOG.info("comparing segmentations")
stats = dict(parameters)
coco_eval = COCOeval(coco_gt, coco_dt, 'segm') # bbox
if catIds:
coco_eval.params.catIds = catIds
#coco_eval.params.iouThrs = [.5:.05:.95]
#coco_eval.params.iouThrs = np.linspace(.3, .95, 14)
coco_eval.params.maxDets = [
None
] # unlimited nr of detections (requires pycocotools#559)
#coco_eval.params.areaRng = [(0, np.inf)] # unlimited region size
#coco_eval.params.areaRngLbl = ['all'] # unlimited region size
# Note: The IoU threshold criterion is inadequate for flat segmentation,
# because over-/undersegmentation can quickly become false negative/positive.
# The pycocotools implementation is especially inadequate, because
# it only counts 1:1 matches (and not even the largest or best-scoring, #564).
# On the other hand, purely pixel-wise measures do not distinguish instances,
# i.e. neighbours can quickly become merged or instances torn apart.
# Our approach therefore does not build on pycocotools for matching
# and aggregation, only for fast IoU calculation. All non-zero pairs
# are considered matches if their intersection over union > 0.5 _or_
# their intersection over either side > 0.5. Matches can thus be n:m.
# Non-matches are counted as well (false positives and false negatives).
# Aggregation uses microaveraging over images. Besides counting segments,
# the pixel areas are counted and averaged (as ratios).
# FIXME: We must differentiate between allowable and non-allowable over/under-segmentation (splits/merges).
# (A region's split is allowable if it flows in the textLineOrder of the respective GT,
# i.e. lines are likely to be either on one side or the other, but not both.
# For top-to-bottom/bottom-to-top regions, vertical splits are allowable.
# For left-to-right/right-to-left regions, horizontal splits are allowable.
# To be sure, we could also validate that explicitly – evaluating both levels at the same time.
# Analogously, a number of regions' merge is allowable if it flows in the textLineOrder
# of them all, and the GT global reading order has no other regions in between.
# For top-to-bottom/bottom-to-top regions, vertical merges are allowable.
# For left-to-right/right-to-left regions, horizontal merges are allowable.
# Again, we could also validate that the overall textline flow is equivalent.)
# This difference can in turn be used to weigh a match pair's score accordingly
# when aggregating. For precision-like scores, we would rule out non-allowable merges
# (by counting them as FP), and for recall-like scores, we would rule out non-allowable splits
# (by counting them as FN).
# We can also weigh these non-allowable cases by their share of height
# (in vertical textLineOrder and horizontal writing) or width
# (in horizontal textLineOrder and vertical writing) which is in disagreement,
# or the share of its textlines that have been split or lost.
# Furthermore, we can weigh matches by the share of non-text regions or fg pixels involved.
coco_eval.evaluate()
# get by-page alignment (ignoring inadequate 1:1 matching by pycocotools)
def get(arg):
return lambda x: x[arg]
numImgs = len(coco_eval.params.imgIds)
numAreas = len(coco_eval.params.areaRng)
for imgind, imgId in enumerate(coco_eval.params.imgIds):
img = coco_gt.imgs[imgId]
pageId = img['file_name']
for catind, catId in enumerate(coco_eval.params.catIds):
cat = coco_gt.cats[catId]
catName = cat['name']
if not catId:
continue
# bypassing COCOeval.evaluateImg, hook onto its results
# (again, we stay at areaRng[0]=all and maxDets[0]=all)
start = catind * numImgs * numAreas
evalimg = coco_eval.evalImgs[start + imgind]
if evalimg is None:
continue # no DT and GT here
# record as dict by pageId / by category
imgstats = stats.setdefault('by-image', dict())
pagestats = imgstats.setdefault(pageId, dict())
# get matches and ious and scores
ious = coco_eval.ious[imgId, catId]
if len(ious):
overlaps_dt, overlaps_gt = ious.nonzero()
else:
overlaps_dt = overlaps_gt = []
# reconstruct score sorting in computeIoU
gt = coco_eval._gts[imgId, catId]
dt = coco_eval._dts[imgId, catId]
dtind = np.argsort([-d['score'] for d in dt], kind='mergesort')
dt = [dt[i] for i in dtind]
matches = list()
gtmatches = dict()
dtmatches = dict()
for dtind, gtind in zip(overlaps_dt, overlaps_gt):
d = dt[dtind]
g = gt[gtind]
iou = ious[dtind, gtind]
union = maskArea(
mergeMasks([g['segmentation'], d['segmentation']]))
intersection = int(iou * union)
# cannot use g or d['area'] here, because mask might be fractional (only-fg) instead of outline
areag = int(maskArea(g['segmentation']))
aread = int(maskArea(d['segmentation']))
iogt = intersection / areag
iodt = intersection / aread
if iou < 0.5 and iogt < 0.5 and iodt < 0.5:
continue
gtmatches.setdefault(gtind, list()).append(dtind)
dtmatches.setdefault(dtind, list()).append(gtind)
matches.append(
(g['id'], d['id'], iogt, iodt, iou, intersection))
pagestats.setdefault('true_positives',
dict()).setdefault(catName,
list()).append({
'GT.ID':
g['segment_id'],
'DT.ID':
d['segment_id'],
'GT.area':
areag,
'DT.area':
aread,
'I.area':
intersection,
'IoGT':
iogt,
'IoDT':
iodt,
'IoU':
iou
})
dtmisses = []
for dtind, d in enumerate(dt):
if dtind in dtmatches:
continue
dtmisses.append((d['id'], maskArea(d['segmentation'])))
pagestats.setdefault('false_positives',
dict()).setdefault(catName,
list()).append({
'DT.ID':
d['segment_id'],
'area':
int(d['area'])
})
gtmisses = []
for gtind, g in enumerate(gt):
if gtind in gtmatches:
continue
gtmisses.append((g['id'], maskArea(g['segmentation'])))
pagestats.setdefault('false_negatives',
dict()).setdefault(catName,
list()).append({
'GT.ID':
g['segment_id'],
'area':
int(g['area'])
})
# measure under/oversegmentation for this image and category
# (follows Zhang et al 2021: Rethinking Semantic Segmentation Evaluation [arXiv:2101.08418])
over_gt = set(gtind for gtind in gtmatches
if len(gtmatches[gtind]) > 1)
over_dt = set(
chain.from_iterable(gtmatches[gtind] for gtind in gtmatches
if len(gtmatches[gtind]) > 1))
under_dt = set(dtind for dtind in dtmatches
if len(dtmatches[dtind]) > 1)
under_gt = set(
chain.from_iterable(dtmatches[dtind] for dtind in dtmatches
if len(dtmatches[dtind]) > 1))
over_degree = sum(len(gtmatches[gtind]) - 1 for gtind in gtmatches)
under_degree = sum(
len(dtmatches[dtind]) - 1 for dtind in dtmatches)
if len(dt) and len(gt):
oversegmentation = len(over_gt) * len(over_dt) / len(gt) / len(
dt)
undersegmentation = len(under_gt) * len(under_dt) / len(
gt) / len(dt)
# Zhang's idea of attenuating the under/oversegmentation ratio with a "penalty"
# to account for the degree of further sub-segmentation is misguided IMHO,
# because its degree term depends on the total number of segments:
# oversegmentation = np.tanh(oversegmentation * over_degree)
# undersegmentation = np.tanh(undersegmentation * under_degree)
pagestats.setdefault('oversegmentation',
dict())[catName] = oversegmentation
pagestats.setdefault('undersegmentation',
dict())[catName] = undersegmentation
pagestats.setdefault(
'precision',
dict())[catName] = (len(dt) - len(dtmisses)) / len(dt)
pagestats.setdefault(
'recall',
dict())[catName] = (len(gt) - len(gtmisses)) / len(gt)
tparea = sum(map(get(5), matches)) # sum(inter)
fparea = sum(map(get(1), dtmisses)) # sum(area)
fnarea = sum(map(get(1), gtmisses)) # sum(area)
if tparea or (fparea and fnarea):
pagestats.setdefault(
'pixel_precision',
dict())[catName] = tparea / (tparea + fparea)
pagestats.setdefault(
'pixel_recall',
dict())[catName] = tparea / (tparea + fnarea)
pagestats.setdefault(
'pixel_iou',
dict())[catName] = tparea / (tparea + fparea + fnarea)
# aggregate per-img/per-cat IoUs for microaveraging
evalimg['matches'] = matches # TP
evalimg['dtMisses'] = dtmisses # FP
evalimg['gtMisses'] = gtmisses # FN
evalimg['dtIdsOver'] = [dt[dtind]['id'] for dtind in over_dt]
evalimg['gtIdsOver'] = [gt[gtind]['id'] for gtind in over_gt]
evalimg['dtIdsUnder'] = [dt[dtind]['id'] for dtind in under_dt]
evalimg['gtIdsUnder'] = [gt[gtind]['id'] for gtind in under_gt]
catstats = stats.setdefault('by-category', dict())
# accumulate our over-/undersegmentation and IoU ratios
numImgs = len(coco_eval.params.imgIds)
numAreas = len(coco_eval.params.areaRng)
for catind, catId in enumerate(coco_eval.params.catIds):
cat = coco_gt.cats[catId]
catstats.setdefault(cat['name'], dict())
start = catind * numImgs * numAreas
# again, we stay at areaRng[0]=all and maxDets[0]=all
evalimgs = [
coco_eval.evalImgs[start + imgind] for imgind in range(numImgs)
]
evalimgs = [img for img in evalimgs if img is not None]
assert all(img['category_id'] == catId for img in evalimgs)
assert all(img['maxDet'] is None for img in evalimgs)
assert all(img['aRng'] == coco_eval.params.areaRng[0]
for img in evalimgs)
if not len(evalimgs):
continue
# again, we can ignore gtIgnore here, because we only look at areaRng[0]=all
# again, we can ignore dtIgnore here, because we only look at maxDet=None
numDTs = sum(len(img['dtIds']) for img in evalimgs)
numGTs = sum(len(img['gtIds']) for img in evalimgs)
overDTs = sum(len(img['dtIdsOver']) for img in evalimgs)
overGTs = sum(len(img['gtIdsOver']) for img in evalimgs)
underDTs = sum(len(img['dtIdsUnder']) for img in evalimgs)
underGTs = sum(len(img['gtIdsUnder']) for img in evalimgs)
numIoUs = sum(len(img['matches']) for img in evalimgs)
numFPs = sum(len(img['dtMisses']) for img in evalimgs)
numFNs = sum(len(img['gtMisses']) for img in evalimgs)
sumIoUs = sum(sum(map(get(4), img['matches']))
for img in evalimgs) # sum(iou)
sumIoGTs = sum(sum(map(get(2), img['matches']))
for img in evalimgs) # sum(iogt)
sumIoDTs = sum(sum(map(get(3), img['matches']))
for img in evalimgs) # sum(iodt)
sumTParea = sum(sum(map(get(5), img['matches']))
for img in evalimgs) # sum(inter)
sumFParea = sum(sum(map(get(1), img['dtMisses']))
for img in evalimgs) # sum(area)
sumFNarea = sum(sum(map(get(1), img['gtMisses']))
for img in evalimgs) # sum(area)
if numDTs and numGTs:
oversegmentation = overDTs * overGTs / numDTs / numGTs
undersegmentation = underDTs * underGTs / numDTs / numGTs
precision = (numDTs - numFPs) / numDTs
recall = (numGTs - numFNs) / numGTs
else:
oversegmentation = undersegmentation = precision = recall = -1
if numIoUs:
iou = sumIoUs / numIoUs
iogt = sumIoGTs / numIoUs
iodt = sumIoDTs / numIoUs
else:
iou = iogt = iodt = -1
if sumTParea or (sumFParea and sumFNarea):
pixel_precision = sumTParea / (sumTParea + sumFParea)
pixel_recall = sumTParea / (sumTParea + sumFNarea)
pixel_iou = sumTParea / (sumTParea + sumFParea + sumFNarea)
else:
pixel_precision = pixel_recall = pixel_iou = -1
catstats[cat['name']]['oversegmentation'] = oversegmentation
catstats[cat['name']]['undersegmentation'] = undersegmentation
catstats[cat['name']]['segment-precision'] = precision
catstats[cat['name']]['segment-recall'] = recall
catstats[cat['name']]['IoGT'] = iogt # i.e. per-match pixel-recall
catstats[cat['name']]['IoDT'] = iodt # i.e. per-match pixel-precision
catstats[cat['name']]['IoU'] = iou # i.e. per-match pixel-jaccardindex
catstats[cat['name']]['pixel-precision'] = pixel_precision
catstats[cat['name']]['pixel-recall'] = pixel_recall
catstats[cat['name']]['pixel-iou'] = pixel_iou
coco_eval.accumulate()
coco_eval.summarize()
statInds = np.ones(12, np.bool)
statInds[7] = False # AR maxDet[1]
statInds[8] = False # AR maxDet[2]
coco_eval.stats = coco_eval.stats[statInds]
stats['scores'] = dict(
zip([
'Average Precision (AP) @[ IoU=0.50:0.95 | area= all ]',
'Average Precision (AP) @[ IoU=0.50 | area= all ]',
'Average Precision (AP) @[ IoU=0.75 | area= all ]',
'Average Precision (AP) @[ IoU=0.50:0.95 | area= small ]',
'Average Precision (AP) @[ IoU=0.50:0.95 | area=medium ]',
'Average Precision (AP) @[ IoU=0.50:0.95 | area= large ]',
'Average Recall (AR) @[ IoU=0.50:0.95 | area= all ]',
'Average Recall (AR) @[ IoU=0.50:0.95 | area= small ]',
'Average Recall (AR) @[ IoU=0.50:0.95 | area=medium ]',
'Average Recall (AR) @[ IoU=0.50:0.95 | area= large ]',
], coco_eval.stats.tolist()))
return stats
item['keypoints'] = res
item['image_id'] = image_id
item['score'] , visible = compute_oks(res, cocoGt.loadAnns(ann_idx))
if len(visible) != 0:
for vis in range(17):
item['keypoints'][3* vis + 2] = visible[vis]
result.append(item)
json.dump(result,fp)
fp.close()
cocoDt = cocoGt.loadRes(write_json)
cocoEval = COCOeval(cocoGt, cocoDt, annType)
cocoEval.params.imgIds = keys
cocoEval.params.catIds = [1]
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
pred = json.load(open(write_json, 'r'))
scores = [ x['score'] for x in pred]
ap50 = compute_ap(scores, 0.5)
print('ap50 is %f' % ap50)
logging.info('ap50 is %f' % ap50)
ap = 0
for i in np.arange(0.5,1, 0.05).tolist():
ap = ap + compute_ap(scores, i)
ap = ap / len(np.arange(0.5, 1 , 0.05).tolist())
print('ap is %f' % ap)
logging.info('ap is %f' % ap)
def infer(model,
path,
detections_file,
resize,
max_size,
batch_size,
mixed_precision=True,
is_master=True,
world=0,
annotations=None,
use_dali=True,
is_validation=False,
verbose=True,
rotated_bbox=False):
'Run inference on images from path'
backend = 'pytorch' if isinstance(model, Model) or isinstance(
model, DDP) else 'tensorrt'
stride = model.module.stride if isinstance(model, DDP) else model.stride
# Create annotations if none was provided
if not annotations:
annotations = tempfile.mktemp('.json')
images = [{
'id': i,
'file_name': f
} for i, f in enumerate(os.listdir(path))]
json.dump({'images': images}, open(annotations, 'w'))
# TensorRT only supports fixed input sizes, so override input size accordingly
if backend == 'tensorrt': max_size = max(model.input_size)
# Prepare dataset
if verbose: print('Preparing dataset...')
if rotated_bbox:
if use_dali:
raise NotImplementedError(
"This repo does not currently support DALI for rotated bbox.")
data_iterator = RotatedDataIterator(path,
resize,
max_size,
batch_size,
stride,
world,
annotations,
training=False)
else:
data_iterator = (DaliDataIterator if use_dali else DataIterator)(
path,
resize,
max_size,
batch_size,
stride,
world,
annotations,
training=False)
if verbose: print(data_iterator)
# Prepare model
if backend is 'pytorch':
# If we are doing validation during training,
# no need to register model with AMP again
if not is_validation:
if torch.cuda.is_available(): model = model.cuda()
model = amp.initialize(model,
None,
opt_level='O2' if mixed_precision else 'O0',
keep_batchnorm_fp32=True,
verbosity=0)
model.eval()
if verbose:
print(' backend: {}'.format(backend))
print(' device: {} {}'.format(
world, 'cpu' if not torch.cuda.is_available() else
'GPU' if world == 1 else 'GPUs'))
print(' batch: {}, precision: {}'.format(
batch_size, 'unknown' if backend is 'tensorrt' else
'mixed' if mixed_precision else 'full'))
print(' BBOX type:', 'rotated' if rotated_bbox else 'axis aligned')
print('Running inference...')
results = []
profiler = Profiler(['infer', 'fw'])
with torch.no_grad():
for i, (data, ids, ratios) in enumerate(data_iterator):
# Forward pass
profiler.start('fw')
scores, boxes, classes = model(
data, rotated_bbox) #Need to add model size (B, 3, W, H)
profiler.stop('fw')
results.append([scores, boxes, classes, ids, ratios])
profiler.bump('infer')
if verbose and (profiler.totals['infer'] > 60
or i == len(data_iterator) - 1):
size = len(data_iterator.ids)
msg = '[{:{len}}/{}]'.format(min((i + 1) * batch_size, size),
size,
len=len(str(size)))
msg += ' {:.3f}s/{}-batch'.format(profiler.means['infer'],
batch_size)
msg += ' (fw: {:.3f}s)'.format(profiler.means['fw'])
msg += ', {:.1f} im/s'.format(batch_size /
profiler.means['infer'])
print(msg, flush=True)
profiler.reset()
# Gather results from all devices
if verbose: print('Gathering results...')
results = [torch.cat(r, dim=0) for r in zip(*results)]
if world > 1:
for r, result in enumerate(results):
all_result = [
torch.ones_like(result, device=result.device)
for _ in range(world)
]
torch.distributed.all_gather(list(all_result), result)
results[r] = torch.cat(all_result, dim=0)
if is_master:
# Copy buffers back to host
results = [r.cpu() for r in results]
# Collect detections
detections = []
processed_ids = set()
for scores, boxes, classes, image_id, ratios in zip(*results):
image_id = image_id.item()
if image_id in processed_ids:
continue
processed_ids.add(image_id)
keep = (scores > 0).nonzero()
scores = scores[keep].view(-1)
if rotated_bbox:
boxes = boxes[keep, :].view(-1, 6)
boxes[:, :4] /= ratios
else:
boxes = boxes[keep, :].view(-1, 4) / ratios
classes = classes[keep].view(-1).int()
for score, box, cat in zip(scores, boxes, classes):
if rotated_bbox:
x1, y1, x2, y2, sin, cos = box.data.tolist()
theta = np.arctan2(sin, cos)
w = x2 - x1 + 1
h = y2 - y1 + 1
seg = rotate_box([x1, y1, w, h, theta])
else:
x1, y1, x2, y2 = box.data.tolist()
cat = cat.item()
if 'annotations' in data_iterator.coco.dataset:
cat = data_iterator.coco.getCatIds()[cat]
this_det = {
'image_id': image_id,
'score': score.item(),
'category_id': cat
}
if rotated_bbox:
this_det['bbox'] = [
x1, y1, x2 - x1 + 1, y2 - y1 + 1, theta
]
this_det['segmentation'] = [seg]
else:
this_det['bbox'] = [x1, y1, x2 - x1 + 1, y2 - y1 + 1]
detections.append(this_det)
if detections:
# Save detections
if detections_file and verbose:
print('Writing {}...'.format(detections_file))
detections = {'annotations': detections}
detections['images'] = data_iterator.coco.dataset['images']
if 'categories' in data_iterator.coco.dataset:
detections['categories'] = [
data_iterator.coco.dataset['categories']
]
if detections_file:
json.dump(detections, open(detections_file, 'w'), indent=4)
# Evaluate model on dataset
if 'annotations' in data_iterator.coco.dataset:
if verbose: print('Evaluating model...')
with redirect_stdout(None):
coco_pred = data_iterator.coco.loadRes(
detections['annotations'])
if rotated_bbox:
coco_eval = COCOeval(data_iterator.coco, coco_pred,
'segm')
else:
coco_eval = COCOeval(data_iterator.coco, coco_pred,
'bbox')
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval.stats
else:
print('No detections!')
def test(
data,
weights=None,
batch_size=32,
imgsz=640,
conf_thres=0.001,
iou_thres=0.6, # for NMS
save_json=False,
single_cls=False,
augment=False,
verbose=False,
model=None,
dataloader=None,
save_dir=Path(''), # for saving images
save_txt=False, # for auto-labelling
save_hybrid=False, # for hybrid auto-labelling
save_conf=False, # save auto-label confidences
plots=True,
wandb_logger=None,
compute_loss=None,
half_precision=True,
is_coco=False):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
set_logging()
device = select_device(opt.device, batch_size=batch_size)
# Directories
save_dir = increment_path(Path(opt.project) / opt.name,
exist_ok=opt.exist_ok) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(
parents=True, exist_ok=True) # make dir
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
gs = max(int(model.stride.max()), 32) # grid size (max stride)
imgsz = check_img_size(imgsz, s=gs) # check img_size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Half
half = device.type != 'cpu' and half_precision # half precision only supported on CUDA
if half:
model.half()
# Configure
model.eval()
if isinstance(data, str):
is_coco = data.endswith('coco.yaml')
with open(data) as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
check_dataset(data) # check
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Logging
log_imgs = 0
if wandb_logger and wandb_logger.wandb:
log_imgs = min(wandb_logger.log_imgs, 100)
# Dataloader
if not training:
if device.type != 'cpu':
model(
torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(
next(model.parameters()))) # run once
task = opt.task if opt.task in (
'train', 'val', 'test') else 'val' # path to train/val/test images
dataloader = create_dataloader(data[task],
imgsz,
batch_size,
gs,
opt,
pad=0.5,
rect=True,
prefix=colorstr(f'{task}: '))[0]
seen = 0
confusion_matrix = ConfusionMatrix(nc=nc)
names = {
k: v
for k, v in enumerate(
model.names if hasattr(model, 'names') else model.module.names)
}
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R',
'[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class, wandb_images = [], [], [], [], []
for batch_i, (img, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=s)):
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
with torch.no_grad():
# Run model
t = time_synchronized()
out, train_out = model(
img, augment=augment) # inference and training outputs
t0 += time_synchronized() - t
# Compute loss
if compute_loss:
loss += compute_loss([x.float() for x in train_out],
targets)[1][:3] # box, obj, cls
# Run NMS
targets[:, 2:] *= torch.Tensor([width, height, width,
height]).to(device) # to pixels
lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)
] if save_hybrid else [] # for autolabelling
t = time_synchronized()
out = non_max_suppression(out,
conf_thres=conf_thres,
iou_thres=iou_thres,
labels=lb,
multi_label=True)
t1 += time_synchronized() - t
# Statistics per image
for si, pred in enumerate(out):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
path = Path(paths[si])
seen += 1
if len(pred) == 0:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Predictions
predn = pred.clone()
scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0],
shapes[si][1]) # native-space pred
# Append to text file
if save_txt:
gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0
]] # normalization gain whwh
for *xyxy, conf, cls in predn.tolist():
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) /
gn).view(-1).tolist() # normalized xywh
line = (cls, *xywh,
conf) if save_conf else (cls,
*xywh) # label format
with open(save_dir / 'labels' / (path.stem + '.txt'),
'a') as f:
f.write(('%g ' * len(line)).rstrip() % line + '\n')
# W&B logging - Media Panel Plots
if len(
wandb_images
) < log_imgs and wandb_logger.current_epoch > 0: # Check for test operation
if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0:
box_data = [{
"position": {
"minX": xyxy[0],
"minY": xyxy[1],
"maxX": xyxy[2],
"maxY": xyxy[3]
},
"class_id": int(cls),
"box_caption": "%s %.3f" % (names[cls], conf),
"scores": {
"class_score": conf
},
"domain": "pixel"
} for *xyxy, conf, cls in pred.tolist()]
boxes = {
"predictions": {
"box_data": box_data,
"class_labels": names
}
} # inference-space
wandb_images.append(
wandb_logger.wandb.Image(img[si],
boxes=boxes,
caption=path.name))
wandb_logger.log_training_progress(
predn, path,
names) if wandb_logger and wandb_logger.wandb_run else None
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(
path.stem) if path.stem.isnumeric() else path.stem
box = xyxy2xywh(predn[:, :4]) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({
'image_id':
image_id,
'category_id':
coco91class[int(p[5])] if is_coco else int(p[5]),
'bbox': [round(x, 3) for x in b],
'score':
round(p[4], 5)
})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0],
niou,
dtype=torch.bool,
device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5])
scale_coords(img[si].shape[1:], tbox, shapes[si][0],
shapes[si][1]) # native-space labels
if plots:
confusion_matrix.process_batch(
predn, torch.cat((labels[:, 0:1], tbox), 1))
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(
-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(
-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(predn[pi, :4], tbox[ti]).max(
1) # best ious, indices
# Append detections
detected_set = set()
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d.item() not in detected_set:
detected_set.add(d.item())
detected.append(d)
correct[
pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(
detected
) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append(
(correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if plots and batch_i < 3:
f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels
Thread(target=plot_images,
args=(img, targets, paths, f, names),
daemon=True).start()
f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions
Thread(target=plot_images,
args=(img, output_to_target(out), paths, f, names),
daemon=True).start()
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats,
plot=plots,
save_dir=save_dir,
names=names)
ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%12i' * 2 + '%12.3g' * 4 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3
for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print(
'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g'
% t)
# Plots
if plots:
confusion_matrix.plot(save_dir=save_dir, names=list(names.values()))
if wandb_logger and wandb_logger.wandb:
val_batches = [
wandb_logger.wandb.Image(str(f), caption=f.name)
for f in sorted(save_dir.glob('test*.jpg'))
]
wandb_logger.log({"Validation": val_batches})
if wandb_images:
wandb_logger.log({"Bounding Box Debugger/Images": wandb_images})
# Save JSON
if save_json and len(jdict):
w = Path(weights[0] if isinstance(weights, list) else weights
).stem if weights is not None else '' # weights
anno_json = '../coco/annotations/instances_val2017.json' # annotations json
pred_json = str(save_dir / f"{w}_predictions.json") # predictions json
print('\nEvaluating pycocotools mAP... saving %s...' % pred_json)
with open(pred_json, 'w') as f:
json.dump(jdict, f)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
anno = COCO(anno_json) # init annotations api
pred = anno.loadRes(pred_json) # init predictions api
eval = COCOeval(anno, pred, 'bbox')
if is_coco:
eval.params.imgIds = [
int(Path(x).stem) for x in dataloader.dataset.img_files
] # image IDs to evaluate
eval.evaluate()
eval.accumulate()
eval.summarize()
map, map50 = eval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print(f'pycocotools unable to run: {e}')
# Return results
model.float() # for training
if not training:
s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
print(f"Results saved to {save_dir}{s}")
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def main():
args = get_args()
cocoGt = COCO(
os.path.join(args.coco_dir, "annotations/instances_val2017.json"))
if args.use_inv_map:
inv_map = [0
] + cocoGt.getCatIds() # First label in inv_map is not used
with open(args.mlperf_accuracy_file, "r") as f:
results = json.load(f)
detections = []
image_ids = set()
seen = set()
image_map = cocoGt.dataset["images"]
for j in results:
idx = j['qsl_idx']
# de-dupe in case loadgen sends the same image multiple times
if idx in seen:
continue
seen.add(idx)
# reconstruct from mlperf accuracy log
# what is written by the benchmark is an array of float32's:
# id, box[0], box[1], box[2], box[3], score, detection_class
# note that id is a index into instances_val2017.json, not the actual image_id
data = np.frombuffer(bytes.fromhex(j['data']), np.float32)
for i in range(0, len(data), 7):
image_idx, ymin, xmin, ymax, xmax, score, label = data[i:i + 7]
image = image_map[idx]
image_id = image["id"]
height, width = image["height"], image["width"]
ymin *= height
xmin *= width
ymax *= height
xmax *= width
loc = os.path.join(args.coco_dir, "val2017", image["file_name"])
label = int(label)
if args.use_inv_map:
label = inv_map[label]
# pycoco wants {imageID,x1,y1,w,h,score,class}
detections.append({
"image_id":
image_id,
"image_loc":
loc,
"category_id":
label,
"bbox": [
float(xmin),
float(ymin),
float(xmax - xmin),
float(ymax - ymin)
],
"score":
float(score)
})
image_ids.add(image_id)
with open(args.output_file, "w") as fp:
json.dump(detections, fp, sort_keys=True, indent=4)
cocoDt = cocoGt.loadRes(
args.output_file) # Load from file to bypass error with Python3
cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox')
cocoEval.params.imgIds = list(image_ids)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("mAP={:.3f}%".format(100. * cocoEval.stats[0]))
if args.verbose:
print("found and ignored {} dupes".format(len(results) - len(seen)))
def evaluate(self, model):
"""
COCO average precision (AP) Evaluation. Iterate inference on the test dataset
and the results are evaluated by COCO API.
Args:
model : model object
Returns:
ap50_95 (float) : calculated COCO AP for IoU=50:95
ap50 (float) : calculated COCO AP for IoU=50
"""
model.eval()
ids = []
data_dict = []
num_images = len(self.dataset)
print('total number of images: %d' % (num_images))
# start testing
for index in range(num_images): # all the data in val2017
if index % 500 == 0:
print('[Eval: %d / %d]' % (index, num_images))
img, id_ = self.dataset.pull_image(index) # load a batch
if self.transform is not None:
x = torch.from_numpy(
self.transform(img)[0][:, :, (2, 1, 0)]).permute(2, 0, 1)
x = x.unsqueeze(0).to(self.device)
scale = np.array(
[[img.shape[1], img.shape[0], img.shape[1], img.shape[0]]])
id_ = int(id_)
ids.append(id_)
with torch.no_grad():
outputs = model(x)
bboxes, scores, cls_inds = outputs
bboxes *= scale
for i, box in enumerate(bboxes):
x1 = float(box[0])
y1 = float(box[1])
x2 = float(box[2])
y2 = float(box[3])
label = self.dataset.class_ids[int(cls_inds[i])]
bbox = [x1, y1, x2 - x1, y2 - y1]
score = float(scores[i]) # object score * class score
A = {
"image_id": id_,
"category_id": label,
"bbox": bbox,
"score": score
} # COCO json format
data_dict.append(A)
annType = ['segm', 'bbox', 'keypoints']
# Evaluate the Dt (detection) json comparing with the ground truth
if len(data_dict) > 0:
print('evaluating ......')
cocoGt = self.dataset.coco
# workaround: temporarily write data to json file because pycocotools can't process dict in py36.
if self.testset:
json.dump(data_dict, open('yolov2_2017.json', 'w'))
cocoDt = cocoGt.loadRes('yolov2_2017.json')
else:
_, tmp = tempfile.mkstemp()
json.dump(data_dict, open(tmp, 'w'))
cocoDt = cocoGt.loadRes(tmp)
cocoEval = COCOeval(self.dataset.coco, cocoDt, annType[1])
cocoEval.params.imgIds = ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
return cocoEval.stats[0], cocoEval.stats[1]
else:
return 0, 0
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
classwise=False,
proposal_nums=(100, 300, 1000),
iou_thrs=np.arange(0.5, 0.96, 0.05)):
"""Evaluation in COCO protocol.
Args:
results (list): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
classwise (bool): Whether to evaluating the AP for each class.
proposal_nums (Sequence[int]): Proposal number used for evaluating
recalls, such as recall@100, recall@1000.
Default: (100, 300, 1000).
iou_thrs (Sequence[float]): IoU threshold used for evaluating
recalls. If set to a list, the average recall of all IoUs will
also be computed. Default: 0.5.
Returns:
dict[str: float]
"""
metrics = metric if isinstance(metric, list) else [metric]
allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast']
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError('metric {} is not supported'.format(metric))
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
eval_results = {}
cocoGt = self.coco
for metric in metrics:
msg = 'Evaluating {}...'.format(metric)
if logger is None:
msg = '\n' + msg
print_log(msg, logger=logger)
if metric == 'proposal_fast':
ar = self.fast_eval_recall(results,
proposal_nums,
iou_thrs,
logger='silent')
log_msg = []
for i, num in enumerate(proposal_nums):
eval_results['AR@{}'.format(num)] = ar[i]
log_msg.append('\nAR@{}\t{:.4f}'.format(num, ar[i]))
log_msg = ''.join(log_msg)
print_log(log_msg, logger=logger)
continue
if metric not in result_files:
raise KeyError('{} is not in results'.format(metric))
try:
cocoDt = cocoGt.loadRes(result_files[metric])
except IndexError:
print_log('The testing results of the whole dataset is empty.',
logger=logger,
level=logging.ERROR)
break
iou_type = 'bbox' if metric == 'proposal' else metric
cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
cocoEval.params.imgIds = self.img_ids
if metric == 'proposal':
cocoEval.params.useCats = 0
cocoEval.params.maxDets = list(proposal_nums)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
metric_items = [
'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000',
'AR_l@1000'
]
for i, item in enumerate(metric_items):
val = float('{:.3f}'.format(cocoEval.stats[i + 6]))
eval_results[item] = val
else:
# cocoEval.params.catIds = [1]
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if classwise: # Compute per-category AP
pass # TODO
metric_items = [
'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l'
]
for i in range(len(metric_items)):
key = '{}_{}'.format(metric, metric_items[i])
val = float('{:.3f}'.format(cocoEval.stats[i]))
eval_results[key] = val
eval_results['{}_mAP_copypaste'.format(metric)] = (
'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} '
'{ap[4]:.3f} {ap[5]:.3f}').format(ap=cocoEval.stats[:6])
if tmp_dir is not None:
tmp_dir.cleanup()
return eval_results
class Coco(Base):
text_labels_keypoints = [
'AP', 'AP0.5', 'AP0.75', 'APM', 'APL', 'AR', 'AR0.5', 'AR0.75', 'ARM',
'ARL'
]
text_labels_bbox = [
'AP', 'AP0.5', 'AP0.75', 'APS', 'APM', 'APL', 'ART1', 'ART10', 'AR',
'ARS', 'ARM', 'ARL'
]
def __init__(self,
coco,
*,
max_per_image=20,
category_ids=None,
iou_type='keypoints',
small_threshold=0.0,
keypoint_oks_sigmas=None):
super().__init__()
if category_ids is None:
category_ids = [1]
self.coco = coco
self.max_per_image = max_per_image
self.category_ids = category_ids
self.iou_type = iou_type
self.small_threshold = small_threshold
self.keypoint_oks_sigmas = keypoint_oks_sigmas
self.predictions = []
self.image_ids = []
self.eval = None
if self.iou_type == 'keypoints':
self.text_labels = self.text_labels_keypoints
elif self.iou_type == 'bbox':
self.text_labels = self.text_labels_bbox
else:
LOG.warning('Unknown iou type "%s". Specify text_labels yourself.',
self.iou_type)
LOG.debug('max = %d, category ids = %s, iou_type = %s',
self.max_per_image, self.category_ids, self.iou_type)
def _stats(self, predictions=None, image_ids=None):
# from pycocotools.cocoeval import COCOeval
if predictions is None:
predictions = self.predictions
if image_ids is None:
image_ids = self.image_ids
coco_eval = self.coco.loadRes(predictions)
self.eval = COCOeval(self.coco, coco_eval, iouType=self.iou_type)
LOG.info('cat_ids: %s', self.category_ids)
if self.category_ids:
self.eval.params.catIds = self.category_ids
if self.keypoint_oks_sigmas is not None:
self.eval.params.kpt_oks_sigmas = np.asarray(
self.keypoint_oks_sigmas)
if image_ids is not None:
print('image ids', image_ids)
self.eval.params.imgIds = image_ids
self.eval.evaluate()
self.eval.accumulate()
self.eval.summarize()
return self.eval.stats
# pylint: disable=unused-argument
def accumulate(self, predictions, image_meta, *, ground_truth=None):
image_id = int(image_meta['image_id'])
self.image_ids.append(image_id)
if self.small_threshold:
predictions = [
pred for pred in predictions
if pred.scale(v_th=0.01) >= self.small_threshold
]
if len(predictions) > self.max_per_image:
predictions = predictions[:self.max_per_image]
image_annotations = []
for pred in predictions:
pred_data = pred.json_data()
pred_data['image_id'] = image_id
pred_data = {
k: v
for k, v in pred_data.items()
if k in ('category_id', 'score', 'keypoints', 'bbox',
'image_id')
}
image_annotations.append(pred_data)
# force at least one annotation per image (for pycocotools)
if not image_annotations:
n_keypoints = (len(self.keypoint_oks_sigmas)
if self.keypoint_oks_sigmas is not None else 17)
image_annotations.append({
'image_id':
image_id,
'category_id':
1,
'keypoints':
np.zeros((n_keypoints * 3, )).tolist(),
'bbox': [0, 0, 1, 1],
'score':
0.001,
})
if LOG.getEffectiveLevel() == logging.DEBUG:
self._stats(image_annotations, [image_id])
LOG.debug(image_meta)
self.predictions += image_annotations
def write_predictions(self, filename, *, additional_data=None):
predictions = [{
k: v
for k, v in annotation.items()
if k in ('image_id', 'category_id', 'keypoints', 'score')
} for annotation in self.predictions]
with open(filename + '.pred.json', 'w') as f:
json.dump(predictions, f)
LOG.info('wrote %s.pred.json', filename)
with zipfile.ZipFile(filename + '.zip', 'w') as myzip:
myzip.write(filename + '.pred.json', arcname='predictions.json')
LOG.info('wrote %s.zip', filename)
if additional_data:
with open(filename + '.pred_meta.json', 'w') as f:
json.dump(additional_data, f)
LOG.info('wrote %s.pred_meta.json', filename)
def stats(self):
data = {
'stats': self._stats().tolist(),
'text_labels': self.text_labels,
}
return data
class InstanceEvaluator(object):
def __init__(self, dataset_json, preds_json):
# load dataset ground truths
self.dataset = COCO(dataset_json)
category_ids = self.dataset.getCatIds()
categories = [x['name'] for x in self.dataset.loadCats(category_ids)]
self.category_to_id_map = dict(zip(categories, category_ids))
self.classes = ['__background__'] + categories
self.num_classes = len(self.classes)
# load predictions
self.preds = self.dataset.loadRes(preds_json)
self.coco_eval = COCOeval(self.dataset, self.preds, 'segm')
self.coco_eval.params.maxDets = [1, 50, 255]
def evaluate(self):
self.coco_eval.evaluate()
self.coco_eval.accumulate()
def _summarize(self, ap=1, iouThr=None, areaRng='all', maxDets=255):
p = self.coco_eval.params
iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'
titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
typeStr = '(AP)' if ap==1 else '(AR)'
iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \
if iouThr is None else '{:0.2f}'.format(iouThr)
aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]
mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
if ap == 1:
# dimension of precision: [TxRxKxAxM]
s = self.coco_eval.eval['precision']
# IoU
if iouThr is not None:
t = np.where(iouThr == p.iouThrs)[0]
s = s[t]
s = s[:,:,:,aind,mind]
else:
# dimension of recall: [TxKxAxM]
s = self.coco_eval.eval['recall']
if iouThr is not None:
t = np.where(iouThr == p.iouThrs)[0]
s = s[t]
s = s[:,:,aind,mind]
if len(s[s>-1])==0:
mean_s = -1
else:
mean_s = np.mean(s[s>-1])
print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))
return mean_s
def summarize(self, IoU_lo_thres=0.5, IoU_hi_thres=0.95):
def _get_thr_ind(thr):
ind = np.where((self.coco_eval.params.iouThrs > thr - 1e-5) &
(self.coco_eval.params.iouThrs < thr + 1e-5))[0][0]
iou_thr = self.coco_eval.params.iouThrs[ind]
assert np.isclose(iou_thr, thr)
return ind
ind_lo = _get_thr_ind(IoU_lo_thres)
ind_hi = _get_thr_ind(IoU_hi_thres)
# (iou, recall, cls, area, max_dets)
precision = self.coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :, :, 0, 2]
ap_mean = np.mean(precision[precision > -1])
print('* MeanAP: {}'.format(ap_mean))
print('* Performance by class:')
ap_by_class = []
for cls_ind, cls_name in enumerate(self.classes):
if cls_name == '__background__':
continue
cls_precision = self.coco_eval.eval['precision'][ind_lo: (ind_hi + 1), :, cls_ind - 1, 0, 2]
cls_ap = np.mean(cls_precision[cls_precision > -1])
ap_by_class.append(cls_ap)
print('{}, AP: {}'.format(cls_name, cls_ap))
ap_by_class = np.asarray(ap_by_class)
print('* Performance at different thresholds:')
ap_by_thres = np.zeros((12,))
ap_by_thres[0] = self._summarize(1)
ap_by_thres[1] = self._summarize(1, iouThr=.5, maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[2] = self._summarize(1, iouThr=.75, maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[3] = self._summarize(1, areaRng='small', maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[4] = self._summarize(1, areaRng='medium', maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[5] = self._summarize(1, areaRng='large', maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[6] = self._summarize(0, maxDets=self.coco_eval.params.maxDets[0])
ap_by_thres[7] = self._summarize(0, maxDets=self.coco_eval.params.maxDets[1])
ap_by_thres[8] = self._summarize(0, maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[9] = self._summarize(0, areaRng='small', maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[10] = self._summarize(0, areaRng='medium', maxDets=self.coco_eval.params.maxDets[2])
ap_by_thres[11] = self._summarize(0, areaRng='large', maxDets=self.coco_eval.params.maxDets[2])
return ap_mean, ap_by_class, ap_by_thres
def do_vg_evaluation(
cfg,
dataset,
predictions,
output_folder,
logger,
iou_types,
):
# get zeroshot triplet
zeroshot_triplet = torch.load(
"maskrcnn_benchmark/data/datasets/evaluation/vg/zeroshot_triplet.pytorch",
map_location=torch.device("cpu")).long().numpy()
attribute_on = cfg.MODEL.ATTRIBUTE_ON
num_attributes = cfg.MODEL.ROI_ATTRIBUTE_HEAD.NUM_ATTRIBUTES
# extract evaluation settings from cfg
# mode = cfg.TEST.RELATION.EVAL_MODE
if cfg.MODEL.ROI_RELATION_HEAD.USE_GT_BOX:
if cfg.MODEL.ROI_RELATION_HEAD.USE_GT_OBJECT_LABEL:
mode = 'predcls'
else:
mode = 'sgcls'
else:
mode = 'sgdet'
num_rel_category = cfg.MODEL.ROI_RELATION_HEAD.NUM_CLASSES
multiple_preds = cfg.TEST.RELATION.MULTIPLE_PREDS
iou_thres = cfg.TEST.RELATION.IOU_THRESHOLD
assert mode in {'predcls', 'sgdet', 'sgcls', 'phrdet', 'preddet'}
groundtruths = []
for image_id, prediction in enumerate(predictions):
img_info = dataset.get_img_info(image_id)
image_width = img_info["width"]
image_height = img_info["height"]
# recover original size which is before transform
predictions[image_id] = prediction.resize((image_width, image_height))
gt = dataset.get_groundtruth(image_id, evaluation=True)
groundtruths.append(gt)
save_output(output_folder, groundtruths, predictions, dataset)
result_str = '\n' + '=' * 100 + '\n'
if "bbox" in iou_types:
# create a Coco-like object that we can use to evaluate detection!
anns = []
for image_id, gt in enumerate(groundtruths):
labels = gt.get_field('labels').tolist() # integer
boxes = gt.bbox.tolist() # xyxy
for cls, box in zip(labels, boxes):
anns.append({
'area': (box[3] - box[1] + 1) * (box[2] - box[0] + 1),
'bbox':
[box[0], box[1], box[2] - box[0] + 1,
box[3] - box[1] + 1], # xywh
'category_id':
cls,
'id':
len(anns),
'image_id':
image_id,
'iscrowd':
0,
})
fauxcoco = COCO()
fauxcoco.dataset = {
'info': {
'description': 'use coco script for vg detection evaluation'
},
'images': [{
'id': i
} for i in range(len(groundtruths))],
'categories': [{
'supercategory': 'person',
'id': i,
'name': name
} for i, name in enumerate(dataset.ind_to_classes)
if name != '__background__'],
'annotations':
anns,
}
fauxcoco.createIndex()
# format predictions to coco-like
cocolike_predictions = []
for image_id, prediction in enumerate(predictions):
box = prediction.convert(
'xywh').bbox.detach().cpu().numpy() # xywh
score = prediction.get_field(
'pred_scores').detach().cpu().numpy() # (#objs,)
label = prediction.get_field(
'pred_labels').detach().cpu().numpy() # (#objs,)
# for predcls, we set label and score to groundtruth
if mode == 'predcls':
label = prediction.get_field('labels').detach().cpu().numpy()
score = np.ones(label.shape[0])
assert len(label) == len(box)
image_id = np.asarray([image_id] * len(box))
cocolike_predictions.append(
np.column_stack((image_id, box, score, label)))
# logger.info(cocolike_predictions)
cocolike_predictions = np.concatenate(cocolike_predictions, 0)
# evaluate via coco API
res = fauxcoco.loadRes(cocolike_predictions)
coco_eval = COCOeval(fauxcoco, res, 'bbox')
coco_eval.params.imgIds = list(range(len(groundtruths)))
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
mAp = coco_eval.stats[1]
result_str += 'Detection evaluation mAp=%.4f\n' % mAp
result_str += '=' * 100 + '\n'
if "relations" in iou_types:
result_dict = {}
evaluator = {}
# tradictional Recall@K
eval_recall = SGRecall(result_dict)
eval_recall.register_container(mode)
evaluator['eval_recall'] = eval_recall
# no graphical constraint
eval_nog_recall = SGNoGraphConstraintRecall(result_dict)
eval_nog_recall.register_container(mode)
evaluator['eval_nog_recall'] = eval_nog_recall
# test on different distribution
eval_zeroshot_recall = SGZeroShotRecall(result_dict)
eval_zeroshot_recall.register_container(mode)
evaluator['eval_zeroshot_recall'] = eval_zeroshot_recall
# test on no graph constraint zero-shot recall
eval_ng_zeroshot_recall = SGNGZeroShotRecall(result_dict)
eval_ng_zeroshot_recall.register_container(mode)
evaluator['eval_ng_zeroshot_recall'] = eval_ng_zeroshot_recall
# used by https://github.com/NVIDIA/ContrastiveLosses4VRD for sgcls and predcls
eval_pair_accuracy = SGPairAccuracy(result_dict)
eval_pair_accuracy.register_container(mode)
evaluator['eval_pair_accuracy'] = eval_pair_accuracy
# used for meanRecall@K
eval_mean_recall = SGMeanRecall(result_dict,
num_rel_category,
dataset.ind_to_predicates,
print_detail=True)
eval_mean_recall.register_container(mode)
evaluator['eval_mean_recall'] = eval_mean_recall
# used for no graph constraint mean Recall@K
eval_ng_mean_recall = SGNGMeanRecall(result_dict,
num_rel_category,
dataset.ind_to_predicates,
print_detail=True)
eval_ng_mean_recall.register_container(mode)
evaluator['eval_ng_mean_recall'] = eval_ng_mean_recall
# prepare all inputs
global_container = {}
global_container['zeroshot_triplet'] = zeroshot_triplet
global_container['result_dict'] = result_dict
global_container['mode'] = mode
global_container['multiple_preds'] = multiple_preds
global_container['num_rel_category'] = num_rel_category
global_container['iou_thres'] = iou_thres
global_container['attribute_on'] = attribute_on
global_container['num_attributes'] = num_attributes
for groundtruth, prediction in zip(groundtruths, predictions):
evaluate_relation_of_one_image(groundtruth, prediction,
global_container, evaluator)
# calculate mean recall
eval_mean_recall.calculate_mean_recall(mode)
eval_ng_mean_recall.calculate_mean_recall(mode)
# print result
result_str += eval_recall.generate_print_string(mode)
result_str += eval_nog_recall.generate_print_string(mode)
result_str += eval_zeroshot_recall.generate_print_string(mode)
result_str += eval_ng_zeroshot_recall.generate_print_string(mode)
result_str += eval_mean_recall.generate_print_string(mode)
result_str += eval_ng_mean_recall.generate_print_string(mode)
if cfg.MODEL.ROI_RELATION_HEAD.USE_GT_BOX:
result_str += eval_pair_accuracy.generate_print_string(mode)
result_str += '=' * 100 + '\n'
logger.info(result_str)
if "relations" in iou_types:
if output_folder:
torch.save(result_dict,
os.path.join(output_folder, 'result_dict.pytorch'))
return float(np.mean(result_dict[mode + '_recall'][100]))
elif "bbox" in iou_types:
return float(mAp)
else:
return -1
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
classwise=False,
proposal_nums=(100, 300, 1000),
iou_thrs=np.arange(0.5, 0.96, 0.05)):
"""Evaluation in COCO protocol.
Args:
results (list): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
classwise (bool): Whether to evaluating the AP for each class.
proposal_nums (Sequence[int]): Proposal number used for evaluating
recalls, such as recall@100, recall@1000.
Default: (100, 300, 1000).
iou_thrs (Sequence[float]): IoU threshold used for evaluating
recalls. If set to a list, the average recall of all IoUs will
also be computed. Default: 0.5.
Returns:
dict[str: float]
"""
metrics = metric if isinstance(metric, list) else [metric]
allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast']
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError(f'metric {metric} is not supported')
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
eval_results = {}
cocoGt = self.coco
for metric in metrics:
msg = f'Evaluating {metric}...'
if logger is None:
msg = '\n' + msg
print_log(msg, logger=logger)
if metric == 'proposal_fast':
ar = self.fast_eval_recall(results,
proposal_nums,
iou_thrs,
logger='silent')
log_msg = []
for i, num in enumerate(proposal_nums):
eval_results[f'AR@{num}'] = ar[i]
log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}')
log_msg = ''.join(log_msg)
print_log(log_msg, logger=logger)
continue
if metric not in result_files:
raise KeyError(f'{metric} is not in results')
try:
cocoDt = cocoGt.loadRes(result_files[metric])
except IndexError:
print_log('The testing results of the whole dataset is empty.',
logger=logger,
level=logging.ERROR)
break
iou_type = 'bbox' if metric == 'proposal' else metric
cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
cocoEval.params.catIds = self.cat_ids
cocoEval.params.imgIds = self.img_ids
if metric == 'proposal':
cocoEval.params.useCats = 0
cocoEval.params.maxDets = list(proposal_nums)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
metric_items = [
'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000',
'AR_l@1000'
]
for i, item in enumerate(metric_items):
val = float(f'{cocoEval.stats[i + 6]:.3f}')
eval_results[item] = val
else:
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if classwise: # Compute per-category AP
# Compute per-category AP
# from https://github.com/facebookresearch/detectron2/
precisions = cocoEval.eval['precision']
# precision: (iou, recall, cls, area range, max dets)
assert len(self.cat_ids) == precisions.shape[2]
results_per_category = []
for idx, catId in enumerate(self.cat_ids):
# area range index 0: all area ranges
# max dets index -1: typically 100 per image
nm = self.coco.loadCats(catId)[0]
precision = precisions[:, :, idx, 0, -1]
precision = precision[precision > -1]
if precision.size:
ap = np.mean(precision)
else:
ap = float('nan')
results_per_category.append(
(f'{nm["name"]}', f'{float(ap):0.3f}'))
num_columns = min(6, len(results_per_category) * 2)
results_flatten = list(
itertools.chain(*results_per_category))
headers = ['category', 'AP'] * (num_columns // 2)
results_2d = itertools.zip_longest(*[
results_flatten[i::num_columns]
for i in range(num_columns)
])
table_data = [headers]
table_data += [result for result in results_2d]
table = AsciiTable(table_data)
print_log('\n' + table.table, logger=logger)
metric_items = [
'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l'
]
for i in range(len(metric_items)):
key = f'{metric}_{metric_items[i]}'
val = float(f'{cocoEval.stats[i]:.3f}')
eval_results[key] = val
ap = cocoEval.stats[:6]
eval_results[f'{metric}_mAP_copypaste'] = (
f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} '
f'{ap[4]:.3f} {ap[5]:.3f}')
if tmp_dir is not None:
tmp_dir.cleanup()
return eval_results
def main(args):
cocoGt = COCO(args.groundtruth_path)
if not args.just_calculate:
# coco format gt
torch.set_default_tensor_type('torch.FloatTensor')
net = network.Network(args)
net.cuda('cuda')
check_point = torch.load(args.model_path)
net.load_state_dict(check_point['state_dict'])
os.makedirs(args.output_dir, exist_ok=True)
retDets = []
for image_id in tqdm(cocoGt.getImgIds()):
img_file_name = cocoGt.loadImgs(ids=[image_id])[0]['file_name']
img_path = join(args.image_dir, img_file_name)
output_image_dir = None
if args.save_image:
output_image_dir = join(args.output_dir, 'predicts')
os.makedirs(output_image_dir, exist_ok=True)
np.set_printoptions(precision=2, suppress=True)
try:
detections = inference_image_path(
args.image_dir,
img_file_name,
net,
category_id=args.category_id)['dtboxes']
except Exception as e:
print('[E]', e)
print('[!] load error ', join(args.image_dir, img_file_name))
continue
if args.save_image:
bboxes, labels = [], []
for res in detections:
score = res['score']
if score < args.confidence:
continue
bbox = res['bbox']
det = {
'image_id': image_id,
'bbox': bbox,
'category_id': args.category_id,
'score': score,
}
retDets.append(det)
if args.save_image:
labels.append(args.category_id)
bboxes.append([
bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1],
score
])
if args.save_image:
dst_img_path = join(output_image_dir, basename(img_path))
if not bboxes:
bboxes = [[0., 0., 0., 0., 0.]]
labels = [args.category_id]
bboxes = np.asarray([
xywh2xyxy(det['bbox']) + [det['score']]
for det in detections
])
labels = np.asarray([det['category_id'] for det in detections])
# formatting
mmcv.imshow_det_bboxes(img_path,
bboxes,
labels,
score_thr=args.confidence,
show=False,
out_file=dst_img_path)
json_dict = {
'images': cocoGt.loadImgs(cocoGt.getImgIds()),
'annotations': retDets,
'categories': [{
'supercategory': 'person',
'id': args.category_id,
}],
}
output_json_path = join(args.output_dir, 'predictions.json')
with open(output_json_path, 'w') as json_fp:
json_str = json.dumps(json_dict)
json_fp.write(json_str)
output_json_path = join(args.output_dir, 'detres.json')
with open(output_json_path, 'w') as json_fp:
json_str = json.dumps(retDets)
json_fp.write(json_str)
if args.calculate:
cocoDt = cocoGt.loadRes(join(args.output_dir, 'detres.json'))
cocoEval = COCOeval(cocoGt, cocoDt, "bbox")
cocoEval.params.maxDets = [100, 500, 1000]
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print('Done')
def benchmark_model(frozen_graph,
images_dir,
annotation_path,
batch_size=1,
image_shape=None,
num_images=4096,
tmp_dir='.benchmark_model_tmp_dir',
remove_tmp_dir=True,
output_path=None):
"""Computes accuracy and performance statistics
Computes accuracy and performance statistics by executing over many images
from the MSCOCO dataset defined by images_dir and annotation_path.
Args
----
frozen_graph: A GraphDef representing the object detection model to
test. Alternatively, a string representing the path to the saved
frozen graph.
images_dir: A string representing the path of the COCO images
directory.
annotation_path: A string representing the path of the COCO annotation
file.
batch_size: An integer representing the batch size to use when feeding
images to the model.
image_shape: An optional tuple of integers representing a fixed shape
to resize all images before testing.
num_images: An integer representing the number of images in the
dataset to evaluate with.
tmp_dir: A string representing the path where the function may create
a temporary directory to store intermediate files.
output_path: An optional string representing a path to store the
statistics in JSON format.
Returns
-------
statistics: A named dictionary of accuracy and performance statistics
computed for the model.
"""
if os.path.exists(tmp_dir):
if not remove_tmp_dir:
raise RuntimeError('Temporary directory exists; %s' % tmp_dir)
subprocess.call(['rm', '-rf', tmp_dir])
if batch_size > 1 and image_shape is None:
raise RuntimeError(
'Fixed image shape must be provided for batch size > 1')
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
coco = COCO(annotation_file=annotation_path)
# get list of image ids to use for evaluation
image_ids = coco.getImgIds()
if num_images > len(image_ids):
print(
'Num images provided %d exceeds number in dataset %d, using %d images instead'
% (num_images, len(image_ids), len(image_ids)))
num_images = len(image_ids)
image_ids = image_ids[0:num_images]
# load frozen graph from file if string, otherwise must be GraphDef
if isinstance(frozen_graph, str):
frozen_graph_path = frozen_graph
frozen_graph = tf.GraphDef()
with open(frozen_graph_path, 'rb') as f:
frozen_graph.ParseFromString(f.read())
elif not isinstance(frozen_graph, tf.GraphDef):
raise TypeError('Expected frozen_graph to be GraphDef or str')
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
coco_detections = [] # list of all bounding box detections in coco format
runtimes = [] # list of runtimes for each batch
image_counts = [] # list of number of images in each batch
with tf.Graph().as_default() as tf_graph:
with tf.Session(config=tf_config) as tf_sess:
tf.import_graph_def(frozen_graph, name='')
tf_input = tf_graph.get_tensor_by_name(INPUT_NAME + ':0')
tf_boxes = tf_graph.get_tensor_by_name(BOXES_NAME + ':0')
tf_classes = tf_graph.get_tensor_by_name(CLASSES_NAME + ':0')
tf_scores = tf_graph.get_tensor_by_name(SCORES_NAME + ':0')
tf_num_detections = tf_graph.get_tensor_by_name(
NUM_DETECTIONS_NAME + ':0')
# load batches from coco dataset
for image_idx in tqdm.tqdm(range(0, len(image_ids), batch_size)):
batch_image_ids = image_ids[image_idx:image_idx + batch_size]
batch_images = []
batch_coco_images = []
# read images from file
for image_id in batch_image_ids:
coco_img = coco.imgs[image_id]
batch_coco_images.append(coco_img)
image_path = os.path.join(images_dir,
coco_img['file_name'])
image = _read_image(image_path, image_shape)
batch_images.append(image)
# run once outside of timing to initialize
if image_idx == 0:
boxes, classes, scores, num_detections = tf_sess.run(
[tf_boxes, tf_classes, tf_scores, tf_num_detections],
feed_dict={tf_input: batch_images})
# execute model and compute time difference
t0 = time.time()
boxes, classes, scores, num_detections = tf_sess.run(
[tf_boxes, tf_classes, tf_scores, tf_num_detections],
feed_dict={tf_input: batch_images})
t1 = time.time()
# log runtime and image count
runtimes.append(float(t1 - t0))
image_counts.append(len(batch_images))
# add coco detections for this batch to running list
batch_coco_detections = []
for i, image_id in enumerate(batch_image_ids):
image_width = batch_coco_images[i]['width']
image_height = batch_coco_images[i]['height']
for j in range(int(num_detections[i])):
bbox = boxes[i][j]
bbox_coco_fmt = [
bbox[1] * image_width, # x0
bbox[0] * image_height, # x1
(bbox[3] - bbox[1]) * image_width, # width
(bbox[2] - bbox[0]) * image_height, # height
]
coco_detection = {
'image_id': image_id,
'category_id': int(classes[i][j]),
'bbox': bbox_coco_fmt,
'score': float(scores[i][j])
}
coco_detections.append(coco_detection)
# write coco detections to file
subprocess.call(['mkdir', '-p', tmp_dir])
coco_detections_path = os.path.join(tmp_dir, 'coco_detections.json')
with open(coco_detections_path, 'w') as f:
json.dump(coco_detections, f)
# compute coco metrics
cocoDt = coco.loadRes(coco_detections_path)
eval = COCOeval(coco, cocoDt, 'bbox')
eval.params.imgIds = image_ids
eval.evaluate()
eval.accumulate()
eval.summarize()
statistics = {
'map': eval.stats[0],
'avg_latency_ms': 1000.0 * np.mean(runtimes),
'avg_throughput_fps': np.sum(image_counts) / np.sum(runtimes)
}
if output_path is not None:
subprocess.call(['mkdir', '-p', os.path.dirname(output_path)])
with open(output_path, 'w') as f:
json.dump(statistics, f)
subprocess.call(['rm', '-rf', tmp_dir])
return statistics
def test(
data,
weights=None,
batch_size=32,
imgsz=640,
conf_thres=0.001,
iou_thres=0.6, # for NMS
save_json=False,
single_cls=False,
augment=False,
verbose=False,
model=None,
dataloader=None,
save_dir=Path(''), # for saving images
save_txt=False, # for auto-labelling
save_hybrid=False, # for hybrid auto-labelling
save_conf=False, # save auto-label confidences
plots=True,
log_imgs=0): # number of logged images
li = []
temp_path = "/content/drive/MyDrive/00Colab Notebooks/07Datasets/COCO/cocomax/"
for i in os.listdir(temp_path):
li.append(cv2.imread(temp_path + i))
logger.info("log:{}{}".format(len(li), li[0].shape))
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
###############コメントアウト
# set_logging()
device = select_device(opt.device, batch_size=batch_size)
# Directories
save_dir = Path(
increment_path(Path(opt.project) / opt.name,
exist_ok=opt.exist_ok)) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(
parents=True, exist_ok=True) # make dir
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Half
half = device.type != 'cpu' # half precision only supported on CUDA
if half:
model.half()
# logger.info("gpu_model:{}".format(gpu_used.gpuinfo()))
# Configure
model.eval()
###############コメントアウト
# is_coco = data.endswith('coco.yaml') # is COCO dataset
with open(data) as f:
data = yaml.load(f, Loader=yaml.FullLoader) # model dict
check_dataset(data) # check
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Logging
log_imgs, wandb = min(log_imgs, 100), None # ceil
try:
import wandb # Weights & Biases
except ImportError:
log_imgs = 0
# Dataloader
if not training:
#下二行、デバック
img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img
_ = model(img.half() if half else img
) if device.type != 'cpu' else None # run once
path = data['test'] if opt.task == 'test' else data[
'val'] # path to val/test images
##################追加
path = "/content/drive/MyDrive/00Colab Notebooks/07Datasets/COCO/cocomax"
logger.info(path)
time_veri.start_point(time_veri.t_point0)
# dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True)[0]
dataloader = create_dataloader(path,
imgsz,
batch_size,
model.stride.max(),
opt,
pad=0.5,
rect=True,
li=li)[0]
time_veri.end_point(time_veri.t_point0)
seen = 0
confusion_matrix = ConfusionMatrix(nc=nc)
names = {
k: v
for k, v in enumerate(
model.names if hasattr(model, 'names') else model.module.names)
}
# coco91class = coco80_to_coco91_class()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R',
'[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1, t_ex1, t_ex2,load_time = 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., []
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class, wandb_images = [], [], [], [], []
# logger.info("gpu_dataloader:{}".format(gpu_used.gpuinfo()))
time_veri.time_start()
for batch_i, (img, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=s)):
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
# gpu_used.gpuinfo()
with torch.no_grad():
# Run model
inf_out, train_out = model(
img, augment=augment) # inference and training outputs
###################################コメントアウト
# # Compute loss
# if training:
# loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # box, obj, cls
# Run NMS
targets[:, 2:] *= torch.Tensor([width, height, width,
height]).to(device) # to pixels
lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)
] if save_hybrid else [] # for autolabelling
output = non_max_suppression(inf_out,
conf_thres=conf_thres,
iou_thres=iou_thres,
labels=lb)
# Statistics per image
for si, pred in enumerate(output):
##############################コメントアウト
# labels = targets[targets[:, 0] == si, 1:]
# nl = len(labels)
# tcls = labels[:, 0].tolist() if nl else [] # target class
# path = Path(paths[si])
# seen += 1
# if len(pred) == 0:
# if nl:
# stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))
# continue
# Predictions→元画像の大きさにリサイズ
predn = pred.clone()
# scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred
########################引数、元座標うけとり
ex_coords = scale_coords(img[si].shape[1:], predn[:, :4],
shapes[si][0],
shapes[si][1]) # native-space pred
###############################追加
# continue
################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################
# # Append to text file
# if save_txt:
# gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh
# for *xyxy, conf, cls in predn.tolist():
# xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh
# line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format
# with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f:
# f.write(('%g ' * len(line)).rstrip() % line + '\n')
# # W&B logging
# if plots and len(wandb_images) < log_imgs:
# box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
# "class_id": int(cls),
# "box_caption": "%s %.3f" % (names[cls], conf),
# "scores": {"class_score": conf},
# "domain": "pixel"} for *xyxy, conf, cls in pred.tolist()]
# boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space
# wandb_images.append(wandb.Image(img[si], boxes=boxes, caption=path.name))
# # Append to pycocotools JSON dictionary
# if save_json:
# # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
# image_id = int(path.stem) if path.stem.isnumeric() else path.stem
# box = xyxy2xywh(predn[:, :4]) # xywh
# box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
# for p, b in zip(pred.tolist(), box.tolist()):
# jdict.append({'image_id': image_id,
# 'category_id': coco91class[int(p[5])] if is_coco else int(p[5]),
# 'bbox': [round(x, 3) for x in b],
# 'score': round(p[4], 5)})
# # Assign all predictions as incorrect
# correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
# if nl:
# detected = [] # target indices
# tcls_tensor = labels[:, 0]
# # target boxes
# tbox = xywh2xyxy(labels[:, 1:5])
# scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels
# if plots:
# confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1))
# # Per target class
# for cls in torch.unique(tcls_tensor):
# ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices
# pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices
# # Search for detections
# if pi.shape[0]:
# # Prediction to target ious
# ious, i = box_iou(predn[pi, :4], tbox[ti]).max(1) # best ious, indices
# # Append detections
# detected_set = set()
# for j in (ious > iouv[0]).nonzero(as_tuple=False):
# d = ti[i[j]] # detected target
# if d.item() not in detected_set:
# detected_set.add(d.item())
# detected.append(d)
# correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn
# if len(detected) == nl: # all targets already located in image
# break
# # Append statistics (correct, conf, pcls, tcls)
# stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################ ################
#####################追加
# time_veri.end_point(time_veri.t_point4)
# time_veri2.start_point(time_veri2.t_point0)
# continue
# # Plot images
# if plots and batch_i < 3:
# f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels
# Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start()
# f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions
# Thread(target=plot_images, args=(img, output_to_target(output), paths, f, names), daemon=True).start()
#####################################追加##################################
time_veri.time_end()
t_alle = time_synchronized()
logger.info("log:{}".format("処理中断終了"))
logger.info("一気通貫:{}".format(time_veri.t_end - time_veri.t_start))
temp = time_veri.t_end - time_veri.t_start
logger.info("fps２２:{}".format(1 / (temp / 500)))
# logger.info("gpu_exit:{}".format(gpu_used.gpuinfo()))
# logger.info("gpu_max,list:{}{}".format(gpu_used.used_max, gpu_used.used_list))
import sys
sys.exit()
#####################################追加##################################
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats,
plot=plots,
save_dir=save_dir,
names=names)
p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(
1) # [P, R, [email protected], [email protected]:0.95]
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%12.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3
for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print(
'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g'
% t)
# Plots
if plots:
confusion_matrix.plot(save_dir=save_dir, names=list(names.values()))
if wandb and wandb.run:
wandb.log({"Images": wandb_images})
wandb.log({
"Validation": [
wandb.Image(str(f), caption=f.name)
for f in sorted(save_dir.glob('test*.jpg'))
]
})
# Save JSON
if save_json and len(jdict):
w = Path(weights[0] if isinstance(weights, list) else weights
).stem if weights is not None else '' # weights
anno_json = '../coco/annotations/instances_val2017.json' # annotations json
pred_json = str(save_dir / f"{w}_predictions.json") # predictions json
print('\nEvaluating pycocotools mAP... saving %s...' % pred_json)
with open(pred_json, 'w') as f:
json.dump(jdict, f)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
anno = COCO(anno_json) # init annotations api
pred = anno.loadRes(pred_json) # init predictions api
eval = COCOeval(anno, pred, 'bbox')
if is_coco:
eval.params.imgIds = [
int(Path(x).stem) for x in dataloader.dataset.img_files
] # image IDs to evaluate
eval.evaluate()
eval.accumulate()
eval.summarize()
map, map50 = eval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print(f'pycocotools unable to run: {e}')
# Return results
if not training:
s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
print(f"Results saved to {save_dir}{s}")
model.float() # for training
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def metrics(pred_data):
"""Calculate mAP of predicted bboxes."""
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
num_classes = config.num_classes
coco_root = config.coco_root
data_type = config.val_data_type
# Classes need to train or test.
val_cls = config.coco_classes
val_cls_dict = {}
for i, cls in enumerate(val_cls):
val_cls_dict[i] = cls
anno_json = os.path.join(coco_root, config.instances_set.format(data_type))
coco_gt = COCO(anno_json)
classs_dict = {}
cat_ids = coco_gt.loadCats(coco_gt.getCatIds())
for cat in cat_ids:
classs_dict[cat["name"]] = cat["id"]
predictions = []
img_ids = []
for sample in pred_data:
pred_boxes = sample['boxes']
box_scores = sample['box_scores']
img_id = sample['img_id']
h, w = sample['image_shape']
pred_boxes = ssd_bboxes_decode(pred_boxes)
final_boxes = []
final_label = []
final_score = []
img_ids.append(img_id)
for c in range(1, num_classes):
class_box_scores = box_scores[:, c]
score_mask = class_box_scores > config.min_score
class_box_scores = class_box_scores[score_mask]
class_boxes = pred_boxes[score_mask] * [h, w, h, w]
if score_mask.any():
nms_index = apply_nms(class_boxes, class_box_scores,
config.nms_thershold, config.max_boxes)
class_boxes = class_boxes[nms_index]
class_box_scores = class_box_scores[nms_index]
final_boxes += class_boxes.tolist()
final_score += class_box_scores.tolist()
final_label += [classs_dict[val_cls_dict[c]]] * \
len(class_box_scores)
for loc, label, score in zip(final_boxes, final_label, final_score):
res = {}
res['image_id'] = img_id
res['bbox'] = [loc[1], loc[0], loc[3] - loc[1], loc[2] - loc[0]]
res['score'] = score
res['category_id'] = label
predictions.append(res)
with open('predictions.json', 'w') as f:
json.dump(predictions, f)
coco_dt = coco_gt.loadRes('predictions.json')
E = COCOeval(coco_gt, coco_dt, iouType='bbox')
E.params.imgIds = img_ids
E.evaluate()
E.accumulate()
E.summarize()
return E.stats[0]
def evaluate_coco(generator, model, threshold=0.05):
""" Use the pycocotools to evaluate a COCO model on a dataset.
Args
generator : The generator for generating the evaluation data.
model : The model to evaluate.
threshold : The score threshold to use.
"""
# start collecting results
results = []
image_ids = []
for index in progressbar.progressbar(range(generator.size()), prefix='COCO evaluation: '):
image = generator.load_image(index)
image = generator.preprocess_image(image)
image, scale = generator.resize_image(image)
if keras.backend.image_data_format() == 'channels_first':
image = image.transpose((2, 0, 1))
# run network
boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))
# correct boxes for image scale
boxes /= scale
# change to (x, y, w, h) (MS COCO standard)
boxes[:, :, 2] -= boxes[:, :, 0]
boxes[:, :, 3] -= boxes[:, :, 1]
# compute predicted labels and scores
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted, so we can break
if score < threshold:
break
# append detection for each positively labeled class
image_result = {
'image_id' : generator.image_ids[index],
'category_id' : generator.label_to_coco_label(label),
'score' : float(score),
'bbox' : box.tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(generator.image_ids[index])
if not len(results):
return
# write output
json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4)
json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4)
# load results in COCO evaluation tool
coco_true = generator.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval.stats
def evaluate_coco(generator, model, threshold=0.05):
# start collecting results
results = []
image_ids = []
for index in range(generator.size()):
image = generator.load_image(index)
image = generator.preprocess_image(image)
image, scale = generator.resize_image(image)
# run network
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(image, axis=0))
# correct boxes for image scale
boxes /= scale
# change to (x, y, w, h) (MS COCO standard)
boxes[:, :, 2] -= boxes[:, :, 0]
boxes[:, :, 3] -= boxes[:, :, 1]
# compute predicted labels and scores
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted, so we can break
if score < threshold:
break
# append detection for each positively labeled class
image_result = {
'image_id': generator.image_ids[index],
'category_id': generator.label_to_coco_label(label),
'score': float(score),
'bbox': box.tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(generator.image_ids[index])
# print progress
print('{}/{}'.format(index, generator.size()), end='\r')
if not len(results):
return
# write output
json.dump(results,
open('{}_bbox_results.json'.format(generator.set_name), 'w'),
indent=4)
json.dump(image_ids,
open('{}_processed_image_ids.json'.format(generator.set_name),
'w'),
indent=4)
# load results in COCO evaluation tool
coco_true = generator.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(
generator.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval.stats
score_threshold=0.001,
iou_threshold=0.45,
method='nms')
for det in out:
x, y, x1, y1, conf, cls = det
detection = {
"image_id": int(data.images[img_id]),
"category_id": int(data._valid_ids[int(cls)]),
"bbox": [x, y, x1 - x, y1 - y],
"score": float("{:.2f}".format(conf))
}
detections.append(detection)
json.dump(detections, open('mAP/results.json', 'w'))
coco_dets = data.coco.loadRes('mAP/results.json')
coco_eval = COCOeval(data.coco, coco_dets, "bbox")
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
##
# Average Precision(AP) @ [IoU = 0.50:0.95 | area = all | maxDets = 100] = 0.343
# Average Precision(AP) @ [IoU = 0.50 | area = all | maxDets = 100] = 0.572
# Average Precision(AP) @ [IoU = 0.75 | area = all | maxDets = 100] = 0.365
# Average Precision(AP) @ [IoU = 0.50:0.95 | area = small | maxDets = 100] = 0.181
# Average Precision(AP) @ [IoU = 0.50:0.95 | area = medium | maxDets = 100] = 0.377
# Average Precision(AP) @ [IoU = 0.50:0.95 | area = large | maxDets = 100] = 0.451
# Average Recall(AR) @ [IoU = 0.50:0.95 | area = all | maxDets = 1] = 0.282
# Average Recall(AR) @ [IoU = 0.50:0.95 | area = all | maxDets = 10] = 0.435
# Average Recall(AR) @ [IoU = 0.50:0.95 | area = all | maxDets = 100] = 0.462
# Average Recall(AR) @ [IoU = 0.50:0.95 | area = small | maxDets = 100] = 0.296
# Average Recall(AR) @ [IoU = 0.50:0.95 | area = medium | maxDets = 100] = 0.490
# Average Recall(AR) @ [IoU = 0.50:0.95 | area = large | maxDets = 100] = 0.573
def test(cfg,
data_cfg,
weights,
batch_size=16,
img_size=416,
iou_thres=0.5,
conf_thres=0.3,
nms_thres=0.45,
save_json=True,
model=None):
device = torch_utils.select_device()
if model is None:
# Initialize model
model = Darknet(cfg, img_size).to(device)
# Load weights
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(
torch.load(weights, map_location=device)['model'])
else: # darknet format
_ = load_darknet_weights(model, weights)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
# Configure run
data_cfg = parse_data_cfg(data_cfg)
nC = int(data_cfg['classes']) # number of classes (80 for COCO)
test_path = data_cfg['valid']
# Dataloader
dataset = LoadImagesAndLabels(test_path, img_size=img_size)
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=4,
pin_memory=False,
collate_fn=dataset.collate_fn)
model.eval()
mean_mAP, mean_R, mean_P, seen = 0.0, 0.0, 0.0, 0
print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP'))
mP, mR, mAPs, TP, jdict = [], [], [], [], []
AP_accum, AP_accum_count = np.zeros(nC), np.zeros(nC)
coco91class = coco80_to_coco91_class()
for imgs, targets, paths, shapes in tqdm(dataloader):
t = time.time()
targets = targets.to(device)
imgs = imgs.to(device)
output = model(imgs)
output = non_max_suppression(output,
conf_thres=conf_thres,
nms_thres=nms_thres)
# Compute average precision for each sample
for si, detections in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
seen += 1
if detections is None:
# If there are labels but no detections mark as zero AP
if len(labels) != 0:
mP.append(0), mR.append(0), mAPs.append(0)
continue
# Get detections sorted by decreasing confidence scores
detections = detections[(-detections[:, 4]).argsort()]
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
box = detections[:, :4].clone() # xyxy
scale_coords(img_size, box, shapes[si]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
# add to json dictionary
for di, d in enumerate(detections):
jdict.append({
'image_id':
int(Path(paths[si]).stem.split('_')[-1]),
'category_id':
coco91class[int(d[6])],
'bbox': [float3(x) for x in box[di]],
'score':
float3(d[4] * d[5])
})
# If no labels add number of detections as incorrect
correct = []
if len(labels) == 0:
# correct.extend([0 for _ in range(len(detections))])
mP.append(0), mR.append(0), mAPs.append(0)
continue
else:
# Extract target boxes as (x1, y1, x2, y2)
target_box = xywh2xyxy(labels[:, 1:5]) * img_size
target_cls = labels[:, 0]
detected = []
for *pred_box, conf, cls_conf, cls_pred in detections:
# Best iou, index between pred and targets
iou, bi = bbox_iou(pred_box, target_box).max(0)
# If iou > threshold and class is correct mark as correct
if iou > iou_thres and cls_pred == target_cls[
bi] and bi not in detected:
correct.append(1)
detected.append(bi)
else:
correct.append(0)
# Compute Average Precision (AP) per class
AP, AP_class, R, P = ap_per_class(
tp=np.array(correct),
conf=detections[:, 4].cpu().numpy(),
pred_cls=detections[:, 6].cpu().numpy(),
target_cls=target_cls.cpu().numpy())
# Accumulate AP per class
AP_accum_count += np.bincount(AP_class, minlength=nC)
AP_accum += np.bincount(AP_class, minlength=nC, weights=AP)
# Compute mean AP across all classes in this image, and append to image list
mP.append(P.mean())
mR.append(R.mean())
mAPs.append(AP.mean())
# Means of all images
mean_P = np.mean(mP)
mean_R = np.mean(mR)
mean_mAP = np.mean(mAPs)
# Print image mAP and running mean mAP
print(('%11s%11s' + '%11.3g' * 4 + 's') %
(seen, len(dataset), mean_P, mean_R, mean_mAP, time.time() - t))
# Print mAP per class
print('\nmAP Per Class:')
for i, c in enumerate(load_classes(data_cfg['names'])):
if AP_accum_count[i]:
print('%15s: %-.4f' % (c, AP_accum[i] / (AP_accum_count[i])))
# Save JSON
if save_json:
imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataset.img_files]
with open('results.json', 'w') as file:
json.dump(jdict, file)
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
# https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
cocoGt = COCO('../coco/annotations/instances_val2014.json'
) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes(
'results.json') # initialize COCO detections api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
# Return mAP
return mean_P, mean_R, mean_mAP
def evaluate_coco(generator, model, threshold=0.05):
# start collecting results
results = []
image_ids = []
for i in range(generator.size()):
image = generator.load_image(i)
image = generator.preprocess_image(image)
image, scale = generator.resize_image(image)
# run network
_, _, detections = model.predict_on_batch(np.expand_dims(image,
axis=0))
# clip to image shape
detections[:, :, 0] = np.maximum(0, detections[:, :, 0])
detections[:, :, 1] = np.maximum(0, detections[:, :, 1])
detections[:, :, 2] = np.minimum(image.shape[1], detections[:, :, 2])
detections[:, :, 3] = np.minimum(image.shape[0], detections[:, :, 3])
# correct boxes for image scale
detections[0, :, :4] /= scale
# change to (x, y, w, h) (MS COCO standard)
detections[:, :, 2] -= detections[:, :, 0]
detections[:, :, 3] -= detections[:, :, 1]
# compute predicted labels and scores
for detection in detections[0, ...]:
positive_labels = np.where(detection[4:] > threshold)[0]
# append detections for each positively labeled class
for label in positive_labels:
image_result = {
'image_id': generator.image_ids[i],
'category_id': generator.label_to_coco_label(label),
'score': float(detection[4 + label]),
'bbox': (detection[:4]).tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(generator.image_ids[i])
# print progress
print('{}/{}'.format(i, generator.size()), end='\r')
if not len(results):
return
# write output
json.dump(results,
open('{}_bbox_results.json'.format(generator.set_name), 'w'),
indent=4)
json.dump(image_ids,
open('{}_processed_image_ids.json'.format(generator.set_name),
'w'),
indent=4)
# load results in COCO evaluation tool
coco_true = generator.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(
generator.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
def evaluate_coco(generator, model, threshold=0.05):
# start collecting results
results = []
image_ids = []
for i in range(generator.size()):
image = generator.load_image(i)
image = generator.preprocess_image(image)
image, scale = generator.resize_image(image)
# run network
_, _, detections = model.predict_on_batch(np.expand_dims(image, axis=0))
# clip to image shape
detections[:, :, 0] = np.maximum(0, detections[:, :, 0])
detections[:, :, 1] = np.maximum(0, detections[:, :, 1])
detections[:, :, 2] = np.minimum(image.shape[1], detections[:, :, 2])
detections[:, :, 3] = np.minimum(image.shape[0], detections[:, :, 3])
# correct boxes for image scale
detections[0, :, :4] /= scale
# change to (x, y, w, h) (MS COCO standard)
detections[:, :, 2] -= detections[:, :, 0]
detections[:, :, 3] -= detections[:, :, 1]
# compute predicted labels and scores
for detection in detections[0, ...]:
positive_labels = np.where(detection[4:] > threshold)[0]
# append detections for each positively labeled class
for label in positive_labels:
image_result = {
'image_id' : generator.image_ids[i],
'category_id' : generator.label_to_coco_label(label),
'score' : float(detection[4 + label]),
'bbox' : (detection[:4]).tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(generator.image_ids[i])
# print progress
print('{}/{}'.format(i, generator.size()), end='\r')
if not len(results):
return
# write output
json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4)
json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4)
# load results in COCO evaluation tool
coco_true = generator.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
def run(
data,
weights=None, # model.pt path(s)
batch_size=32, # batch size
imgsz=640, # inference size (pixels)
conf_thres=0.001, # confidence threshold
iou_thres=0.6, # NMS IoU threshold
task='val', # train, val, test, speed or study
device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
workers=8, # max dataloader workers (per RANK in DDP mode)
single_cls=False, # treat as single-class dataset
augment=False, # augmented inference
verbose=False, # verbose output
save_txt=False, # save results to *.txt
save_hybrid=False, # save label+prediction hybrid results to *.txt
save_conf=False, # save confidences in --save-txt labels
save_json=False, # save a COCO-JSON results file
project=ROOT / 'runs/val', # save to project/name
name='exp', # save to project/name
exist_ok=False, # existing project/name ok, do not increment
half=True, # use FP16 half-precision inference
dnn=False, # use OpenCV DNN for ONNX inference
model=None,
dataloader=None,
save_dir=Path(''),
plots=True,
callbacks=Callbacks(),
compute_loss=None,
):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device, pt, jit, engine = next(model.parameters(
)).device, True, False, False # get model device, PyTorch model
half &= device.type != 'cpu' # half precision only supported on CUDA
model.half() if half else model.float()
else: # called directly
device = select_device(device, batch_size=batch_size)
# Directories
save_dir = increment_path(Path(project) / name,
exist_ok=exist_ok) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(
parents=True, exist_ok=True) # make dir
# Load model
model = DetectMultiBackend(weights,
device=device,
dnn=dnn,
data=data,
fp16=half)
stride, pt, jit, engine = model.stride, model.pt, model.jit, model.engine
imgsz = check_img_size(imgsz, s=stride) # check image size
half = model.fp16 # FP16 supported on limited backends with CUDA
if engine:
batch_size = model.batch_size
else:
device = model.device
if not (pt or jit):
batch_size = 1 # export.py models default to batch-size 1
LOGGER.info(
f'Forcing --batch-size 1 square inference (1,3,{imgsz},{imgsz}) for non-PyTorch models'
)
# Data
data = check_dataset(data) # check
# Configure
model.eval()
cuda = device.type != 'cpu'
is_coco = isinstance(data.get('val'), str) and data['val'].endswith(
f'coco{os.sep}val2017.txt') # COCO dataset
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95, 10,
device=device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if not training:
if pt and not single_cls: # check --weights are trained on --data
ncm = model.model.nc
assert ncm == nc, f'{weights[0]} ({ncm} classes) trained on different --data than what you passed ({nc} ' \
f'classes). Pass correct combination of --weights and --data that are trained together.'
model.warmup(imgsz=(1 if pt else batch_size, 3, imgsz,
imgsz)) # warmup
pad = 0.0 if task in ('speed', 'benchmark') else 0.5
rect = False if task == 'benchmark' else pt # square inference for benchmarks
task = task if task in (
'train', 'val', 'test') else 'val' # path to train/val/test images
dataloader = create_dataloader(data[task],
imgsz,
batch_size,
stride,
single_cls,
pad=pad,
rect=rect,
workers=workers,
prefix=colorstr(f'{task}: '))[0]
seen = 0
confusion_matrix = ConfusionMatrix(nc=nc)
names = {
k: v
for k, v in enumerate(
model.names if hasattr(model, 'names') else model.module.names)
}
class_map = coco80_to_coco91_class() if is_coco else list(range(1000))
s = ('%20s' + '%11s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R',
'[email protected]', '[email protected]:.95')
dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0,
0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
callbacks.run('on_val_start')
pbar = tqdm(dataloader,
desc=s,
bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}') # progress bar
for batch_i, (im, targets, paths, shapes) in enumerate(pbar):
callbacks.run('on_val_batch_start')
t1 = time_sync()
if cuda:
im = im.to(device, non_blocking=True)
targets = targets.to(device)
im = im.half() if half else im.float() # uint8 to fp16/32
im /= 255 # 0 - 255 to 0.0 - 1.0
nb, _, height, width = im.shape # batch size, channels, height, width
t2 = time_sync()
dt[0] += t2 - t1
# Inference
out, train_out = model(im) if training else model(
im, augment=augment, val=True) # inference, loss outputs
dt[1] += time_sync() - t2
# Loss
if compute_loss:
loss += compute_loss([x.float() for x in train_out],
targets)[1] # box, obj, cls
# NMS
targets[:, 2:] *= torch.tensor((width, height, width, height),
device=device) # to pixels
lb = [targets[targets[:, 0] == i, 1:]
for i in range(nb)] if save_hybrid else [] # for autolabelling
t3 = time_sync()
out = non_max_suppression(out,
conf_thres,
iou_thres,
labels=lb,
multi_label=True,
agnostic=single_cls)
dt[2] += time_sync() - t3
# Metrics
for si, pred in enumerate(out):
labels = targets[targets[:, 0] == si, 1:]
nl, npr = labels.shape[0], pred.shape[
0] # number of labels, predictions
path, shape = Path(paths[si]), shapes[si][0]
correct = torch.zeros(npr, niou, dtype=torch.bool,
device=device) # init
seen += 1
if npr == 0:
if nl:
stats.append((correct, *torch.zeros(
(3, 0), device=device)))
continue
# Predictions
if single_cls:
pred[:, 5] = 0
predn = pred.clone()
scale_coords(im[si].shape[1:], predn[:, :4], shape,
shapes[si][1]) # native-space pred
# Evaluate
if nl:
tbox = xywh2xyxy(labels[:, 1:5]) # target boxes
scale_coords(im[si].shape[1:], tbox, shape,
shapes[si][1]) # native-space labels
labelsn = torch.cat((labels[:, 0:1], tbox),
1) # native-space labels
correct = process_batch(predn, labelsn, iouv)
if plots:
confusion_matrix.process_batch(predn, labelsn)
stats.append((correct, pred[:, 4], pred[:, 5],
labels[:, 0])) # (correct, conf, pcls, tcls)
# Save/log
if save_txt:
save_one_txt(predn,
save_conf,
shape,
file=save_dir / 'labels' / (path.stem + '.txt'))
if save_json:
save_one_json(predn, jdict, path,
class_map) # append to COCO-JSON dictionary
callbacks.run('on_val_image_end', pred, predn, path, names, im[si])
# Plot images
if plots and batch_i < 3:
plot_images(im, targets, paths,
save_dir / f'val_batch{batch_i}_labels.jpg',
names) # labels
plot_images(im, output_to_target(out), paths,
save_dir / f'val_batch{batch_i}_pred.jpg',
names) # pred
callbacks.run('on_val_batch_end')
# Compute metrics
stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats,
plot=plots,
save_dir=save_dir,
names=names)
ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%11i' * 2 + '%11.3g' * 4 # print format
LOGGER.info(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
LOGGER.info(pf %
(names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3 for x in dt) # speeds per image
if not training:
shape = (batch_size, 3, imgsz, imgsz)
LOGGER.info(
f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}'
% t)
# Plots
if plots:
confusion_matrix.plot(save_dir=save_dir, names=list(names.values()))
callbacks.run('on_val_end')
# Save JSON
if save_json and len(jdict):
w = Path(weights[0] if isinstance(weights, list) else weights
).stem if weights is not None else '' # weights
anno_json = str(
Path(data.get('path', '../coco')) /
'annotations/instances_val2017.json') # annotations json
pred_json = str(save_dir / f"{w}_predictions.json") # predictions json
LOGGER.info(f'\nEvaluating pycocotools mAP... saving {pred_json}...')
with open(pred_json, 'w') as f:
json.dump(jdict, f)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
check_requirements(['pycocotools'])
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
anno = COCO(anno_json) # init annotations api
pred = anno.loadRes(pred_json) # init predictions api
eval = COCOeval(anno, pred, 'bbox')
if is_coco:
eval.params.imgIds = [
int(Path(x).stem) for x in dataloader.dataset.im_files
] # image IDs to evaluate
eval.evaluate()
eval.accumulate()
eval.summarize()
map, map50 = eval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
LOGGER.info(f'pycocotools unable to run: {e}')
# Return results
model.float() # for training
if not training:
s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}")
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(dataset, model, threshold=0.05):
model.eval()
device = [p.device for p in model.parameters()][0]
with torch.no_grad():
# start collecting results
results = []
image_ids = []
for index in range(len(dataset)):
data = dataset[index]
scale = data['scale']
# run network
scores, labels, boxes = model(data['img'].permute(
2, 0, 1).to(device=device).float().unsqueeze(dim=0))
scores = scores.cpu()
labels = labels.cpu()
boxes = boxes.cpu()
# correct boxes for image scale
boxes /= scale
if boxes.shape[0] > 0:
# change to (x, y, w, h) (MS COCO standard)
boxes[:, 2] -= boxes[:, 0]
boxes[:, 3] -= boxes[:, 1]
# compute predicted labels and scores
#for box, score, label in zip(boxes[0], scores[0], labels[0]):
for box_id in range(boxes.shape[0]):
score = float(scores[box_id])
label = int(labels[box_id])
box = boxes[box_id, :]
# scores are sorted, so we can break
if score < threshold:
break
# append detection for each positively labeled class
image_result = {
'image_id': dataset.image_ids[index],
'category_id': dataset.label_to_coco_label(label),
'score': float(score),
'bbox': box.tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(dataset.image_ids[index])
# print progress
print('{}/{}'.format(index, len(dataset)), end='\r')
if not len(results):
return
# write output
json.dump(results,
open('{}_bbox_results.json'.format(dataset.set_name), 'w'),
indent=4)
# load results in COCO evaluation tool
coco_true = dataset.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(
dataset.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
model.train()
return
def test(cfg,
data_cfg,
weights=None,
batch_size=1,
img_size=416,
iou_thres=0.4,
conf_thres=0.01,
nms_thres=0.4,
save_json=False,
model=None):
if model is None:
print("model is None,重新创建model")
device = torch_utils.select_device()
# Initialize model
model = Darknet(cfg, img_size).to(device)
# Load weights
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(
torch.load(weights, map_location=device)['model'])
else: # darknet format
_ = load_darknet_weights(model, weights)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
else:
device = next(model.parameters()).device # get model device
# Configure run
data_cfg = parse_data_cfg(data_cfg)
nc = int(data_cfg['classes']) # number of classes
test_path = data_cfg['test'] # path to test images
names = load_classes(data_cfg['names']) # class names
# Dataloader
print("***---测试图片路径：---***", test_path)
dataset = LoadImagesAndLabels(test_path,
img_size,
batch_size,
augment=False,
rect=False)
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=12,
pin_memory=True,
collate_fn=dataset.collate_fn)
seen = 0
model.eval()
coco91class = coco80_to_coco91_class()
print(('%30s' + '%10s' * 6) %
('Class', 'Images', 'Targets', 'P', 'R', 'mAP', 'F1'))
loss, p, r, f1, mp, mr, map, mf1 = 0., 0., 0., 0., 0., 0., 0., 0.
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (imgs, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc='Computing mAP')):
targets = targets.to(device)
imgs = imgs.to(device)
_, _, height, width = imgs.shape # batch size, channels, height, width
# Plot images with bounding boxes
if batch_i == 0 and not os.path.exists('test_batch0.jpg'):
plot_images(imgs=imgs,
targets=targets,
paths=paths,
fname='test_batch0.jpg')
# Run model
inf_out, train_out = model(imgs) # inference and training outputs
# Compute loss
if hasattr(model, 'hyp'): # if model has loss hyperparameters
cur_loss = compute_loss(train_out, targets, model)[0].item()
loss += cur_loss
# Run NMS
output = non_max_suppression(inf_out,
conf_thres=conf_thres,
nms_thres=nms_thres)
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append(([], torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
# with open('test.txt', 'a') as file:
# [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred]
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
scale_coords(imgs[si].shape[1:], box,
shapes[si]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for di, d in enumerate(pred):
jdict.append({
'image_id': image_id,
'category_id': coco91class[int(d[6])],
'bbox': [float3(x) for x in box[di]],
'score': float(d[4])
})
# Assign all predictions as incorrect
correct = [0] * len(pred)
if nl:
detected = []
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5])
tbox[:, [0, 2]] *= width
tbox[:, [1, 3]] *= height
# Search for correct predictions
for i, (*pbox, pconf, pcls_conf, pcls) in enumerate(pred):
# Break if all targets already located in image
if len(detected) == nl:
break
# Continue if predicted class not among image classes
if pcls.item() not in tcls:
continue
# Best iou, index between pred and targets
m = (pcls == tcls_tensor).nonzero().view(-1)
iou, bi = bbox_iou(pbox, tbox[m]).max(0)
# If iou > threshold and class is correct mark as correct
if iou > iou_thres and m[
bi] not in detected: # and pcls == tcls[bi]:
correct[i] = 1
detected.append(m[bi])
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct, pred[:, 4].cpu(), pred[:, 6].cpu(), tcls))
# Compute statistics
stats = [np.concatenate(x, 0) for x in list(zip(*stats))] # to numpy
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
if len(stats):
p, r, ap, f1, ap_class = ap_per_class(*stats)
mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()
# Print results
pf = '%30s' + '%10.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1))
# Print results per class
if nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))
# Save JSON
if save_json and map and len(jdict):
imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataset.img_files]
with open('results.json', 'w') as file:
json.dump(jdict, file)
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
# https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
cocoGt = COCO('../coco/annotations/instances_val2014.json'
) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map = cocoEval.stats[1] # update mAP to pycocotools mAP
# Return results
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map, mf1, loss / len(dataloader)), maps
def eval_mscoco_with_segm(cocoGT, cocoPred):
# running evaluation
cocoEval = COCOeval(cocoGT, cocoPred, "keypoints")
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def eval(dataset, predictor):
gt_coco = {
"licenses": {
"name": "",
"id": 0,
"url": ""
},
"images": [],
"annotations": [],
"categories": []
}
for i, c in enumerate(dataset.class_names):
gt_coco["categories"].append(create_coco_category(i, c))
dt_coco = {
"licenses": gt_coco["licenses"],
"annotations": [],
"categories": gt_coco["categories"]
}
input_bbox_converter = BboxFormatConvert(source_format=dataset.bbox_format,
target_format='coco')
output_bbox_converter = BboxFormatConvert(source_format='pascal_voc',
target_format='coco')
for i in interactive(range(len(dataset))):
sample = dataset[i]
image = sample['image']
height, width = image.shape[:2]
image_record = create_coco_image_record(i, (width, height))
gt_coco["images"].append(image_record)
coco_sample = input_bbox_converter(**sample)
boxes = coco_sample['bboxes']
labels = coco_sample['category_id']
scores = [1 for _ in labels]
gt_anns = create_coco_annotations(i, boxes, labels, scores)
gt_coco["annotations"].extend(gt_anns)
boxes, labels, probs = predictor.predict(image, prob_threshold=0)
boxes = [b.tolist() for b in boxes]
labels = labels.tolist()
probs = probs.tolist()
boxes = output_bbox_converter(image=image, bboxes=boxes)["bboxes"]
dt_anns = create_coco_annotations(i, boxes, labels, probs)
dt_coco["annotations"].extend(dt_anns)
dt_coco.update({"images": gt_coco["images"]})
gt_coco_obj = COCO()
gt_coco_obj.dataset = gt_coco
gt_coco_obj.createIndex()
dt_coco_obj = COCO()
dt_coco_obj.dataset = dt_coco
dt_coco_obj.createIndex()
eval = COCOeval(gt_coco_obj, dt_coco_obj, iouType='bbox')
eval.evaluate()
eval.accumulate()
eval.summarize()
result = {
"Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ]":
eval.stats[0],
"Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ]":
eval.stats[1],
"Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ]":
eval.stats[2],
"Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ]":
eval.stats[3],
"Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ]":
eval.stats[4],
"Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ]":
eval.stats[5],
"Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ]":
eval.stats[6],
"Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ]":
eval.stats[7],
"Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ]":
eval.stats[8],
"Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ]":
eval.stats[9],
"Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ]":
eval.stats[10],
"Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ]":
eval.stats[11]
}
return result
from fast_rcnn.nms_wrapper import nms from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval import numpy as np import skimage.io as io import pylab if __name__ == '__main__': pylab.rcParams['figure.figsize'] = (10.0, 8.0) annType = 'bbox' ground_truth = '/mnt/d/BigData/COCO/instances_train-val2014/annotations/instances_val2014.json' generated_result = '/mnt/c/Users/Lavenger/git/py-faster-rcnn/tools/result.json' cocoGt = COCO(generated_result) cocoDt = cocoGt.loadRes(generated_result) cocoEval = COCOeval(cocoGt,cocoDt) cocoEval.params.imgIds = imgIds cocoEval.params.useSegm = False cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def test(
data,
weights=None,
batch_size=16,
imgsz=640,
conf_thres=0.001,
iou_thres=0.6, # for NMS
save_json=False,
single_cls=False,
augment=False,
verbose=False,
model=None,
dataloader=None,
save_dir='',
merge=False,
save_txt=False):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
set_logging()
device = select_device(opt.device, batch_size=batch_size)
merge, save_txt = opt.merge, opt.save_txt # use Merge NMS, save *.txt labels
if save_txt:
out = Path('inference/output')
if os.path.exists(out):
shutil.rmtree(out) # delete output folder
os.makedirs(out) # make new output folder
# Remove previous
for f in glob.glob(str(Path(save_dir) / 'test_batch*.jpg')):
os.remove(f)
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Half
half = device.type != 'cpu' # half precision only supported on CUDA
if half:
model.half()
# Configure
model.eval()
with open(data) as f:
data = yaml.load(f, Loader=yaml.FullLoader) # model dict
check_dataset(data) # check
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if not training:
img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img
_ = model(img.half() if half else img
) if device.type != 'cpu' else None # run once
path = data['test'] if opt.task == 'test' else data[
'val'] # path to val/test images
dataloader = create_dataloader(path,
imgsz,
batch_size,
model.stride.max(),
opt,
hyp=None,
augment=False,
cache=False,
pad=0.5,
rect=True)[0]
seen = 0
names = model.names if hasattr(model, 'names') else model.module.names
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R',
'[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (img, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=s)):
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Disable gradients
with torch.no_grad():
# Run model
t = time_synchronized()
inf_out, train_out = model(
img, augment=augment) # inference and training outputs
t0 += time_synchronized() - t
# Compute loss
if training: # if model has loss hyperparameters
loss += compute_loss([x.float() for x in train_out], targets,
model)[1][:3] # GIoU, obj, cls
# Run NMS
t = time_synchronized()
output = non_max_suppression(inf_out,
conf_thres=conf_thres,
iou_thres=iou_thres,
merge=merge)
t1 += time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
if save_txt:
gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0
]] # normalization gain whwh
x = pred.clone()
x[:, :4] = scale_coords(img[si].shape[1:], x[:, :4],
shapes[si][0],
shapes[si][1]) # to original
for *xyxy, conf, cls in x:
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) /
gn).view(-1).tolist() # normalized xywh
with open(str(out / Path(paths[si]).stem) + '.txt',
'a') as f:
f.write(
('%g ' * 5 + '\n') % (cls, *xywh)) # label format
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = Path(paths[si]).stem
box = pred[:, :4].clone() # xyxy
scale_coords(img[si].shape[1:], box, shapes[si][0],
shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({
'image_id':
int(image_id) if image_id.isnumeric() else image_id,
'category_id':
coco91class[int(p[5])],
'bbox': [round(x, 3) for x in b],
'score':
round(p[4], 5)
})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0],
niou,
dtype=torch.bool,
device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(
-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(
-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(pred[pi, :4], tbox[ti]).max(
1) # best ious, indices
# Append detections
detected_set = set()
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d.item() not in detected_set:
detected_set.add(d.item())
detected.append(d)
correct[
pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(
detected
) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append(
(correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if batch_i < 1:
f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i) # filename
plot_images(img, targets, paths, str(f), names) # ground truth
f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i)
plot_images(img, output_to_target(output, width, height), paths,
str(f), names) # predictions
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats)
p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(
1) # [P, R, [email protected], [email protected]:0.95]
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%12.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3
for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print(
'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g'
% t)
# Save JSON
if save_json and len(jdict):
f = 'detections_val2017_%s_results.json' % \
(weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '') # filename
print('\nCOCO mAP with pycocotools... saving %s...' % f)
with open(f, 'w') as file:
json.dump(jdict, file)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files]
cocoGt = COCO(
glob.glob('../coco/annotations/instances_val*.json')
[0]) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes(f) # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # image IDs to evaluate
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map, map50 = cocoEval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print('ERROR: pycocotools unable to run: %s' % e)
# Return results
model.float() # for training
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def test(cfg,
data,
weights=None,
batch_size=16,
img_size=416,
conf_thres=0.001,
iou_thres=0.6, # for nms
save_json=False,
single_cls=False,
model=None,
dataloader=None):
# Initialize/load model and set device
if model is None:
device = torch_utils.select_device(opt.device, batch_size=batch_size)
verbose = opt.task == 'test'
# Remove previous
for f in glob.glob('test_batch*.png'):
os.remove(f)
# Initialize model
model = Darknet(cfg, img_size).to(device)
# Load weights
attempt_download(weights)
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(torch.load(weights, map_location=device)['model'])
else: # darknet format
load_darknet_weights(model, weights)
if device.type != 'cpu' and torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
else: # called by train.py
device = next(model.parameters()).device # get model device
verbose = False
# Configure run
data = parse_data_cfg(data)
nc = 1 if single_cls else int(data['classes']) # number of classes
path = data['valid'] # path to test images
names = load_classes(data['names']) # class names
iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95
iouv = iouv[0].view(1) # comment for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if dataloader is None:
dataset = LoadImagesAndLabels(path, img_size, batch_size, rect=True, single_cls=opt.single_cls)
batch_size = min(batch_size, len(dataset))
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]),
pin_memory=True,
collate_fn=dataset.collate_fn)
seen = 0
model.eval()
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'F1')
p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):
imgs = imgs.to(device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
_, _, height, width = imgs.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Plot images with bounding boxes
f = 'test_batch%g.png' % batch_i # filename
if batch_i < 1 and not os.path.exists(f):
plot_images(imgs=imgs, targets=targets, paths=paths, fname=f)
# Disable gradients
with torch.no_grad():
# Run model
t = torch_utils.time_synchronized()
inf_out, train_out = model(imgs) # inference and training outputs
t0 += torch_utils.time_synchronized() - t
# Compute loss
if hasattr(model, 'hyp'): # if model has loss hyperparameters
loss += compute_loss(train_out, targets, model)[1][:3] # GIoU, obj, cls
# Run NMS
t = torch_utils.time_synchronized()
output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres) # nms
t1 += torch_utils.time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
# with open('test.txt', 'a') as file:
# [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred]
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
scale_coords(imgs[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for di, d in enumerate(pred):
jdict.append({'image_id': image_id,
'category_id': coco91class[int(d[5])],
'bbox': [floatn(x, 3) for x in box[di]],
'score': floatn(d[4], 5)})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero().view(-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero().view(-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious,_,_= box_iou(pred[pi, :4], tbox[ti])
ious, i = ious.max(1) # best ious, indices
# Append detections
for j in (ious > iouv[0]).nonzero():
d = ti[i[j]] # detected target
if d not in detected:
detected.append(d)
correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(detected) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats):
p, r, ap, f1, ap_class = ap_per_class(*stats)
if niou > 1:
p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]]
mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()
nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))
# Print speeds
if verbose:
t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (img_size, img_size, batch_size) # tuple
print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)
# Save JSON
if save_json and map and len(jdict):
print('\nCOCO mAP with pycocotools...')
imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataloader.dataset.img_files]
with open('results.json', 'w') as file:
json.dump(jdict, file)
try:
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
except:
print('WARNING: missing pycocotools package, can not compute official COCO mAP. See requirements.txt.')
# https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
mf1, map = cocoEval.stats[:2] # update to pycocotools results ([email protected]:0.95, [email protected])
# Return results
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map, mf1, *(loss.cpu() / len(dataloader)).tolist()), maps
def evaluate_coco(dataset, model, is_cuda, save_folder, threshold=0.05):
model.eval()
with torch.no_grad():
# start collecting results
results = []
image_ids = []
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in range(len(dataset))]
for _ in range(coco['num_classes'])]
# timers
_t = {'im_detect': Timer(), 'misc': Timer()}
# for test, kaidong
#pdb.set_trace()
# for test, kaidong
#for index in range(60):
for index in range(len(dataset)):
img, gt, h, w = dataset.pull_item(index)
#data = dataset[index]
x = Variable(img.unsqueeze(0))
if is_cuda:
x = x.cuda()
_t['im_detect'].tic()
detections = model(x).data
detect_time = _t['im_detect'].toc(average=False)
for j in range(1, detections.size(1)):
det = detections[0, j]
index_valid = det[:, 0].gt(0.)
if index_valid.sum() < 1:
continue
det = det[index_valid]
det[:, 1] *= w
det[:, 3] *= w
det[:, 2] *= h
det[:, 4] *= h
# change to coco format, kaidong
det[:, 3] -= det[:, 1]
det[:, 4] -= det[:, 2]
for k in range(det.size(0)):
score = det[k, 0]
box = det[k, 1:]
# append detection for each positively labeled class
image_result = {
'image_id':
dataset.ids[index],
'category_id':
dataset.target_transform.inverse_label_map[j],
#'category_id' : COCO_CLASSES[j-1],
'score':
float(score),
'bbox':
box.tolist(),
}
# for test, kaidong
#if score > 0.5:
# print('coco eval', 'res', image_result)
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(dataset.ids[index])
print('im_detect: {:d}/{:d} {:.3f}s'.format(
index + 1, len(dataset), detect_time))
if not len(results):
return
file_json = os.path.join(
save_folder, '{}_bbox_results.json'.format(dataset.img_set))
# write output
json.dump(results, open(file_json, 'w'), indent=4)
# load results in COCO evaluation tool
coco_true = dataset.coco
coco_pred = coco_true.loadRes(file_json)
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
model.train()
return
def evaluate(net,
dataset,
max_num=-1,
during_training=False,
benchmark=False,
cocoapi=False,
traditional_nms=False):
frame_times = MovingAverage()
dataset_size = len(dataset) if max_num < 0 else min(max_num, len(dataset))
dataset_indices = list(range(len(dataset)))
dataset_indices = dataset_indices[:dataset_size]
progress_bar = ProgressBar(40, dataset_size)
if benchmark:
timer.disable('Data loading')
else:
# For each class and iou, stores tuples (score, isPositive)
# Index ap_data[type][iouIdx][classIdx]
ap_data = {
'box': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds],
'mask': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds]
}
make_json = Make_json()
for i, image_idx in enumerate(dataset_indices):
timer.reset()
with timer.env('Data loading'):
img, gt, gt_masks, h, w, num_crowd = dataset.pull_item(image_idx)
batch = Variable(img.unsqueeze(0))
if cuda:
batch = batch.cuda()
with timer.env('Network forward'):
net_outs = net(batch)
nms_outs = NMS(net_outs, traditional_nms)
if benchmark:
prep_benchmark(nms_outs, h, w)
else:
prep_metrics(ap_data, nms_outs, gt, gt_masks, h, w, num_crowd,
dataset.ids[image_idx], make_json, cocoapi)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
fps = 0
if i > 1 and not during_training:
frame_times.add(timer.total_time())
fps = 1 / frame_times.get_avg()
progress = (i + 1) / dataset_size * 100
progress_bar.set_val(i + 1)
print('\rProcessing: %s %d / %d (%.2f%%) %.2f fps ' %
(repr(progress_bar), i + 1, dataset_size, progress, fps),
end='')
if benchmark:
print('\n\nStats for the last frame:')
timer.print_stats()
avg_seconds = frame_times.get_avg()
print('Average: %5.2f fps, %5.2f ms' %
(1 / frame_times.get_avg(), 1000 * avg_seconds))
else:
if cocoapi:
make_json.dump()
print(
f'\nJson files dumped, saved in: {json_path}, start evaluting.'
)
gt_annotations = COCO(cfg.dataset.valid_info)
bbox_dets = gt_annotations.loadRes(
f'{json_path}/bbox_detections.json')
mask_dets = gt_annotations.loadRes(
f'{json_path}/mask_detections.json')
print('\nEvaluating BBoxes:')
bbox_eval = COCOeval(gt_annotations, bbox_dets, 'bbox')
bbox_eval.evaluate()
bbox_eval.accumulate()
bbox_eval.summarize()
print('\nEvaluating Masks:')
bbox_eval = COCOeval(gt_annotations, mask_dets, 'segm')
bbox_eval.evaluate()
bbox_eval.accumulate()
bbox_eval.summarize()
return
table, mask_row = calc_map(ap_data)
print(table)
return table, mask_row
def main(args):
# create model
model = network.__dict__[cfg.model](cfg.output_shape, cfg.num_skeleton, pretrained = False)
model = torch.nn.DataParallel(model).cuda()
test_loader = torch.utils.data.DataLoader(
Mscoco(cfg, is_train=False),
batch_size=args.batch, shuffle=False,
num_workers=args.workers, pin_memory=True)
# load training weights
checkpoint_file = os.path.join(args.checkpoint, args.test+'.pth.tar')
checkpoint = torch.load(checkpoint_file)
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}' (epoch {})".format(checkpoint_file, checkpoint['epoch']))
# switch to evaluation mode
model.eval()
print('testing...')
full_result = []
for i, (inputs, meta) in tqdm(enumerate(test_loader)):
with torch.no_grad():
input_var = torch.autograd.Variable(inputs.cuda())
# flip the input image
if args.flip == True:
flip_inputs = inputs.clone()
for i, finp in enumerate(flip_inputs):
finp = im_to_numpy(finp)
finp = cv2.flip(finp, 1)
flip_inputs[i] = im_to_torch(finp)
flip_input_var = torch.autograd.Variable(flip_inputs.cuda())
# compute output
global_outputs, refine_output = model(input_var)
score_map = refine_output.data.cpu()
score_map = score_map.numpy()
if args.flip == True:
flip_global_outputs, flip_output = model(flip_input_var)
flip_score_map = flip_output.data.cpu()
flip_score_map = flip_score_map.numpy()
for i, fscore in enumerate(flip_score_map):
fscore = fscore.transpose((1,2,0))
fscore = cv2.flip(fscore, 1)
fscore = list(fscore.transpose((2,0,1)))
for (q, w) in cfg.symmetry:
fscore[q], fscore[w] = fscore[w], fscore[q]
fscore = np.array(fscore)
score_map[i] += fscore
score_map[i] /= 2
ids = meta['img_id'].numpy()
det_scores = meta['det_scores']
for b in range(inputs.size(0)):
details = meta['augmentation_details']
single_result_dict = {}
single_result = []
single_map = score_map[b]
r0 = single_map.copy()
r0 /= 255
r0 += 0.5
v_score = np.zeros(17)
for p in range(17):
single_map[p] /= np.amax(single_map[p])
border = 10
dr = np.zeros((cfg.output_shape[0] + 2*border, cfg.output_shape[1]+2*border))
dr[border:-border, border:-border] = single_map[p].copy()
dr = cv2.GaussianBlur(dr, (21, 21), 0)
lb = dr.argmax()
y, x = np.unravel_index(lb, dr.shape)
dr[y, x] = 0
lb = dr.argmax()
py, px = np.unravel_index(lb, dr.shape)
y -= border
x -= border
py -= border + y
px -= border + x
ln = (px ** 2 + py ** 2) ** 0.5
delta = 0.25
if ln > 1e-3:
x += delta * px / ln
y += delta * py / ln
x = max(0, min(x, cfg.output_shape[1] - 1))
y = max(0, min(y, cfg.output_shape[0] - 1))
resy = float((4 * y + 2) / cfg.data_shape[0] * (details[b][3] - details[b][1]) + details[b][1])
resx = float((4 * x + 2) / cfg.data_shape[1] * (details[b][2] - details[b][0]) + details[b][0])
v_score[p] = float(r0[p, int(round(y)+1e-10), int(round(x)+1e-10)])
single_result.append(resx)
single_result.append(resy)
single_result.append(1)
if len(single_result) != 0:
single_result_dict['image_id'] = int(ids[b])
single_result_dict['category_id'] = 1
single_result_dict['keypoints'] = single_result
single_result_dict['score'] = float(det_scores[b])*v_score.mean()
full_result.append(single_result_dict)
result_path = args.result
if not isdir(result_path):
mkdir_p(result_path)
result_file = os.path.join(result_path, 'result.json')
with open(result_file,'w') as wf:
json.dump(full_result, wf)
# evaluate on COCO
eval_gt = COCO(cfg.ori_gt_path)
eval_dt = eval_gt.loadRes(result_file)
cocoEval = COCOeval(eval_gt, eval_dt, iouType='keypoints')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
