def evaluate():
cocoGt = COCO('annotations.json')
cocoDt = cocoGt.loadRes('detections.json')
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def coco_evaluate(json_dataset, res_file, image_ids):
coco_dt = json_dataset.COCO.loadRes(str(res_file))
coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval
def compute_ap(self):
coco_res = self.loader.coco.loadRes(self.filename)
cocoEval = COCOeval(self.loader.coco, coco_res)
cocoEval.params.imgIds = self.loader.get_filenames()
cocoEval.params.useSegm = False
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
return cocoEval
def _do_coco_eval(self, dtFile, output_dir):
"""
Evaluate using COCO API
"""
if self._image_set == 'train' or self._image_set == 'val':
cocoGt = self._coco[0]
cocoDt = COCO(dtFile)
E = COCOeval(cocoGt, cocoDt)
E.evaluate()
E.accumulate()
E.summarize()
def evaluate_detections(self, all_boxes, output_dir=None):
resFile = self._write_coco_results_file(all_boxes)
cocoGt = self._annotations
cocoDt = cocoGt.loadRes(resFile)
# running evaluation
cocoEval = COCOeval(cocoGt,cocoDt)
# useSegm should default to 0
#cocoEval.params.useSegm = 0
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def cocoval(detected_json):
eval_json = config.eval_json
eval_gt = COCO(eval_json)
eval_dt = eval_gt.loadRes(detected_json)
cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox')
# cocoEval.params.imgIds = eval_gt.getImgIds()
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def _do_keypoint_eval(json_dataset, res_file, output_dir):
ann_type = 'keypoints'
imgIds = json_dataset.COCO.getImgIds()
imgIds.sort()
coco_dt = json_dataset.COCO.loadRes(res_file)
coco_eval = COCOeval(json_dataset.COCO, coco_dt, ann_type)
coco_eval.params.imgIds = imgIds
coco_eval.evaluate()
coco_eval.accumulate()
eval_file = os.path.join(output_dir, 'keypoint_results.pkl')
robust_pickle_dump(coco_eval, eval_file)
logger.info('Wrote json eval results to: {}'.format(eval_file))
coco_eval.summarize()
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
# Pick COCO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
# Get corresponding COCO image IDs.
coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
# Load image
image = dataset.load_image(image_id)
# Run detection
t = time.time()
r = model.detect([image], verbose=0)[0]
t_prediction += (time.time() - t)
# Convert results to COCO format
# Cast masks to uint8 because COCO tools errors out on bool
image_results = build_coco_results(dataset, coco_image_ids[i:i + 1],
r["rois"], r["class_ids"],
r["scores"],
r["masks"].astype(np.uint8))
results.extend(image_results)
# Load results. This modifies results with additional attributes.
coco_results = coco.loadRes(results)
# Evaluate
cocoEval = COCOeval(coco, coco_results, eval_type)
cocoEval.params.imgIds = coco_image_ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
def validate(val_loader, model, i, silence=False):
batch_time = AverageMeter()
coco_gt = val_loader.dataset.coco
coco_pred = COCO()
coco_pred.dataset['images'] = [img for img in coco_gt.datasets['images']]
coco_pred.dataset['categories'] = copy.deepcopy(coco_gt.dataset['categories'])
id = 0
# switch to evaluate mode
model.eval()
end = time.time()
for i, (inputs, anns) in enumerate(val_loader):
# forward images one by one (TODO: support batch mode later, or
# multiprocess)
for j, input in enumerate(inputs):
input_anns= anns[j] # anns of this input
gt_bbox= np.vstack([ann['bbox'] + [ann['ordered_id']] for ann in input_anns])
im_info= [[input.size(1), input.size(2),
input_anns[0]['scale_ratio']]]
input_var= Variable(input.unsqueeze(0),
requires_grad=False).cuda()
cls_prob, bbox_pred, rois = model(input_var, im_info)
scores, pred_boxes = model.interpret_outputs(cls_prob, bbox_pred, rois, im_info)
print(scores, pred_boxes)
# for i in range(scores.shape[0]):
# measure elapsed time
batch_time.update(time.time() - end)
end= time.time()
coco_pred.createIndex()
coco_eval = COCOeval(coco_gt, coco_pred, 'bbox')
coco_eval.params.imgIds= sorted(coco_gt.getImgIds())
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
print('iter: [{0}] '
'Time {batch_time.avg:.3f} '
'Val Stats: {1}'
.format(i, coco_eval.stats,
batch_time=batch_time))
return coco_eval.stats[0]
def evaluate_predictions_on_coco(
coco_gt, coco_results, json_result_file, iou_type="bbox"
):
import json
with open(json_result_file, "w") as f:
json.dump(coco_results, f)
from pycocotools.cocoeval import COCOeval
coco_dt = coco_gt.loadRes(str(json_result_file))
# coco_dt = coco_gt.loadRes(coco_results)
coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval
def calc_coco_metrics(coco_annotations, predictions, classes):
annotations = ObjectDetectorJson.convert_coco_to_toolbox_format(coco_annotations, classes)
detections = []
for annotation, prediction in zip(annotations, predictions):
width, height = annotation['image_size']
image_id = annotation['image_id']
for obj_id, obj in enumerate(prediction):
label = int(obj[1])
score = float(obj[2])
if obj_id != 0 and score == 0: # At least one prediction must be (COCO API issue)
continue
bbox = (obj[3:]).tolist()
bbox[::2] = [width * i for i in bbox[::2]]
bbox[1::2] = [height * i for i in bbox[1::2]]
xmin, ymin, xmax, ymax = bbox
w_bbox = round(xmax - xmin, 1)
h_bbox = round(ymax - ymin, 1)
xmin, ymin = round(xmin, 1), round(ymin, 1)
coco_det = {}
coco_det['image_id'] = image_id
coco_det['category_id'] = label
coco_det['bbox'] = [xmin, ymin, w_bbox, h_bbox]
coco_det['score'] = score
detections.append(coco_det)
coco_dt = coco_annotations.loadRes(detections)
img_ids = sorted(coco_annotations.getImgIds())
coco_eval = COCOeval(coco_annotations, coco_dt, 'bbox')
coco_eval.params.imgIds = img_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
metrics = {}
for metric_name, value in zip(METRICS_NAMES, coco_eval.stats):
metrics[metric_name] = value
return metrics
def print_evaluation_scores(json_file):
ret = {}
assert config.BASEDIR and os.path.isdir(config.BASEDIR)
annofile = os.path.join(
config.BASEDIR, 'annotations',
'instances_{}.json'.format(config.VAL_DATASET))
coco = COCO(annofile)
cocoDt = coco.loadRes(json_file)
cocoEval = COCOeval(coco, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
ret['mAP(bbox)'] = cocoEval.stats[0]
if config.MODE_MASK:
cocoEval = COCOeval(coco, cocoDt, 'segm')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
ret['mAP(segm)'] = cocoEval.stats[0]
return ret
def evaluate_coco(model, dataset, coco, config, eval_type="bbox", limit=None, image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
"""
# Pick COCO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
# Get corresponding COCO image IDs.
coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
if i%10==0:
print('Processed %d images'%i )
# Load image
image = dataset.load_image(image_id)
# Run detection
t = time.time()
r = inference(image, model, config)
t_prediction += (time.time() - t)
# Convert results to COCO format
image_results = build_coco_results(dataset, coco_image_ids[i:i + 1],
r["rois"], r["class_ids"],
r["scores"], r["masks"])
results.extend(image_results)
# Load results. This modifies results with additional attributes.
coco_results = coco.loadRes(results)
# Evaluate
cocoEval = COCOeval(coco, coco_results, eval_type)
cocoEval.params.imgIds = coco_image_ids
# Only evaluate for person.
cocoEval.params.catIds = coco.getCatIds(catNms=['person'])
cocoEval.evaluate()
a=cocoEval.accumulate()
b=cocoEval.summarize()
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
def print_evaluation_scores(json_file):
ret = {}
assert config.BASEDIR and os.path.isdir(config.BASEDIR)
annofile = os.path.join(
config.BASEDIR, 'annotations',
'instances_{}.json'.format(config.VAL_DATASET))
coco = COCO(annofile)
cocoDt = coco.loadRes(json_file)
cocoEval = COCOeval(coco, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
fields = ['IoU=0.5:0.95', 'IoU=0.5', 'IoU=0.75', 'small', 'medium', 'large']
for k in range(6):
ret['mAP(bbox)/' + fields[k]] = cocoEval.stats[k]
if config.MODE_MASK:
cocoEval = COCOeval(coco, cocoDt, 'segm')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
for k in range(6):
ret['mAP(segm)/' + fields[k]] = cocoEval.stats[k]
return ret
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output', default='output', type=str)
parser.add_argument('--data', default='val2017', type=str)
parser.add_argument('--annotations', default='annotations', type=str)
parser.add_argument('--inres', default='512,512', type=str)
parser.add_argument('--no-full-resolution', action='store_true')
args, _ = parser.parse_known_args()
args.inres = tuple(int(x) for x in args.inres.split(','))
if not args.no_full_resolution:
args.inres = (None, None)
os.makedirs(args.output, exist_ok=True)
kwargs = {
'num_stacks': 2,
'cnv_dim': 256,
'weights': 'ctdet_coco',
'inres': args.inres,
}
heads = {'hm': 80, 'reg': 2, 'wh': 2}
out_fn_box = os.path.join(
args.output, args.data + '_bbox_results_%s_%s.json' %
(args.inres[0], args.inres[1]))
model = HourglassNetwork(heads=heads, **kwargs)
model = CtDetDecode(model)
if args.no_full_resolution:
letterbox_transformer = LetterboxTransformer(args.inres[0],
args.inres[1])
else:
letterbox_transformer = LetterboxTransformer(mode='testing',
max_stride=128)
fns = sorted(glob(os.path.join(args.data, '*.jpg')))
results = []
for fn in tqdm(fns):
img = cv2.imread(fn)
image_id = int(os.path.splitext(os.path.basename(fn))[0])
pimg = letterbox_transformer(img)
pimg = normalize_image(pimg)
pimg = np.expand_dims(pimg, 0)
detections = model.predict(pimg)[0]
for d in detections:
x1, y1, x2, y2, score, cl = d
# if score < 0.001:
# break
x1, y1, x2, y2 = letterbox_transformer.correct_box(x1, y1, x2, y2)
cl = int(cl)
x1, y1, x2, y2 = float(x1), float(y1), float(x2), float(y2)
image_result = {
'image_id': image_id,
'category_id': COCO_IDS[cl + 1],
'score': float(score),
'bbox': [x1, y1, (x2 - x1), (y2 - y1)],
}
results.append(image_result)
if not len(results):
print("No predictions were generated.")
return
# write output
with open(out_fn_box, 'w') as f:
json.dump(results, f, indent=2)
print("Predictions saved to: %s" % out_fn_box)
# load results in COCO evaluation tool
gt_fn = os.path.join(args.annotations, 'instances_%s.json' % args.data)
print("Loading GT: %s" % gt_fn)
coco_true = COCO(gt_fn)
coco_pred = coco_true.loadRes(out_fn_box)
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval.stats
def evaluate(config):
is_training = False
# Load and initialize network
net = ModelMain(config, is_training=is_training)
net.train(is_training)
# Set data parallel
net = nn.DataParallel(net)
net = net.cuda()
# Restore pretrain model
if config["pretrain_snapshot"]:
logging.info("Load checkpoint: {}".format(config["pretrain_snapshot"]))
state_dict = torch.load(config["pretrain_snapshot"])
net.load_state_dict(state_dict)
else:
logging.warning("missing pretrain_snapshot!!!")
# YOLO loss with 3 scales
yolo_losses = []
for i in range(3):
yolo_losses.append(
YOLOLoss(config["yolo"]["anchors"][i], config["yolo"]["classes"],
(config["img_w"], config["img_h"])))
# DataLoader.
dataloader = torch.utils.data.DataLoader(COCODataset(
config["val_path"], (config["img_w"], config["img_h"]),
is_training=False),
batch_size=config["batch_size"],
shuffle=False,
num_workers=8,
pin_memory=False)
# Coco Prepare.
index2category = json.load(open("coco_index2category.json"))
# Start the eval loop
logging.info("Start eval.")
coco_results = []
coco_img_ids = set([])
for step, samples in enumerate(dataloader):
images, labels = samples["image"], samples["label"]
image_paths, origin_sizes = samples["image_path"], samples[
"origin_size"]
with torch.no_grad():
outputs = net(images)
output_list = []
for i in range(3):
output_list.append(yolo_losses[i](outputs[i]))
output = torch.cat(output_list, 1)
batch_detections = non_max_suppression(output,
config["yolo"]["classes"],
conf_thres=0.001,
nms_thres=0.45)
for idx, detections in enumerate(batch_detections):
image_id = int(os.path.basename(image_paths[idx])[-16:-4])
coco_img_ids.add(image_id)
if detections is not None:
origin_size = eval(origin_sizes[idx])
detections = detections.cpu().numpy()
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
x1 = x1 / config["img_w"] * origin_size[0]
x2 = x2 / config["img_w"] * origin_size[0]
y1 = y1 / config["img_h"] * origin_size[1]
y2 = y2 / config["img_h"] * origin_size[1]
w = x2 - x1
h = y2 - y1
coco_results.append({
"image_id":
image_id,
"category_id":
index2category[str(int(cls_pred.item()))],
"bbox": (float(x1), float(y1), float(w), float(h)),
"score":
float(conf),
})
logging.info("Now {}/{}".format(step, len(dataloader)))
save_results_path = "coco_results.json"
with open(save_results_path, "w") as f:
json.dump(coco_results,
f,
sort_keys=True,
indent=4,
separators=(',', ':'))
logging.info("Save coco format results to {}".format(save_results_path))
# COCO api
logging.info("Using coco-evaluate tools to evaluate.")
cocoGt = COCO(config["annotation_path"])
cocoDt = cocoGt.loadRes(save_results_path)
cocoEval = COCOeval(cocoGt, cocoDt, "bbox")
cocoEval.params.imgIds = list(coco_img_ids) # real imgIds
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def evaluate_coco(generator, model, threshold=0.05):
# start collecting results
results = []
image_ids = []
for i in range(generator.size()):
image = generator.load_image(i)
image = generator.preprocess_image(image)
image, scale = generator.resize_image(image)
# run network
_, _, detections = model.predict_on_batch(np.expand_dims(image, axis=0))
# clip to image shape
detections[:, :, 0] = np.maximum(0, detections[:, :, 0])
detections[:, :, 1] = np.maximum(0, detections[:, :, 1])
detections[:, :, 2] = np.minimum(image.shape[1], detections[:, :, 2])
detections[:, :, 3] = np.minimum(image.shape[0], detections[:, :, 3])
# correct boxes for image scale
detections[0, :, :4] /= scale
# change to (x, y, w, h) (MS COCO standard)
detections[:, :, 2] -= detections[:, :, 0]
detections[:, :, 3] -= detections[:, :, 1]
# compute predicted labels and scores
for detection in detections[0, ...]:
positive_labels = np.where(detection[4:] > threshold)[0]
# append detections for each positively labeled class
for label in positive_labels:
image_result = {
'image_id' : generator.image_ids[i],
'category_id' : generator.label_to_coco_label(label),
'score' : float(detection[4 + label]),
'bbox' : (detection[:4]).tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(generator.image_ids[i])
# print progress
print('{}/{}'.format(i, generator.size()), end='\r')
if not len(results):
return
# write output
json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4)
json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4)
# load results in COCO evaluation tool
coco_true = generator.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
def test(cfg,
data,
weights=None,
batch_size=16,
imgsz=416,
conf_thres=0.001,
iou_thres=0.6, # for nms
save_json=False,
single_cls=False,
augment=False,
model=None,
dataloader=None,
multi_label=True):
# Initialize/load model and set device
if model is None:
device = torch_utils.select_device(opt.device, batch_size=batch_size)
verbose = opt.task == 'test'
# Remove previous
for f in glob.glob('test_batch*.jpg'):
os.remove(f)
# Initialize model
model = Darknet(cfg, imgsz)
# Load weights
attempt_download(weights)
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(torch.load(weights, map_location=device)['model'])
else: # darknet format
load_darknet_weights(model, weights)
# Fuse
model.fuse()
model.to(device)
if device.type != 'cpu' and torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
else: # called by train.py
device = next(model.parameters()).device # get model device
verbose = False
# Configure run
data = parse_data_cfg(data)
nc = 1 if single_cls else int(data['classes']) # number of classes
path = data['valid'] # path to test images
names = load_classes(data['names']) # class names
iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95
iouv = iouv[0].view(1) # comment for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if dataloader is None:
dataset = LoadImagesAndLabels(path, imgsz, batch_size, rect=True, single_cls=opt.single_cls)
batch_size = min(batch_size, len(dataset))
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]),
pin_memory=True,
collate_fn=dataset.collate_fn)
seen = 0
model.eval()
_ = model(torch.zeros((1, 3, imgsz, imgsz), device=device)) if device.type != 'cpu' else None # run once
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'F1')
p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s, ncols=150)):
imgs = imgs.to(device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = imgs.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Disable gradients
with torch.no_grad():
# Run model
t = torch_utils.time_synchronized()
inf_out, train_out = model(imgs, augment=augment) # inference and training outputs
t0 += torch_utils.time_synchronized() - t
# Compute loss
if hasattr(model, 'hyp'): # if model has loss hyperparameters
loss += compute_loss(train_out, targets, model)[1][:3] # GIoU, obj, cls
# Run NMS
t = torch_utils.time_synchronized()
output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, multi_label=multi_label)
t1 += torch_utils.time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
# with open('test.txt', 'a') as file:
# [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred]
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
scale_coords(imgs[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({'image_id': image_id,
'category_id': coco91class[int(p[5])],
'bbox': [round(x, 3) for x in b],
'score': round(p[4], 5)})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero().view(-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero().view(-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1) # best ious, indices
# Append detections
for j in (ious > iouv[0]).nonzero():
d = ti[i[j]] # detected target
if d not in detected:
detected.append(d)
correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(detected) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if batch_i < 1:
f = 'test_batch%g_gt.jpg' % batch_i # filename
plot_images(imgs, targets, paths=paths, names=names, fname=f) # ground truth
f = 'test_batch%g_pred.jpg' % batch_i
plot_images(imgs, output_to_target(output, width, height), paths=paths, names=names, fname=f) # predictions
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats):
p, r, ap, f1, ap_class = ap_per_class(*stats)
if niou > 1:
p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]]
mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()
nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))
# Print speeds
if verbose or save_json:
t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)
# Save JSON
if save_json and map and len(jdict):
print('\nCOCO mAP with pycocotools...')
imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataloader.dataset.img_files]
with open('results.json', 'w') as file:
json.dump(jdict, file)
try:
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
# https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
# mf1, map = cocoEval.stats[:2] # update to pycocotools results ([email protected]:0.95, [email protected])
except:
print('WARNING: pycocotools must be installed with numpy==1.17 to run correctly. '
'See https://github.com/cocodataset/cocoapi/issues/356')
# Return results
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map, mf1, *(loss.cpu() / len(dataloader)).tolist()), maps
def doEval(self, json_file):
cocoDt = self.coco.loadRes(json_file)
cocoEval = COCOeval(self.coco, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def test(cfg,
data,
weights=None,
batch_size=16,
img_size=416,
iou_thres=0.5,
conf_thres=0.001,
nms_thres=0.5,
save_json=False,
model=None):
# Initialize/load model and set device
if model is None:
device = torch_utils.select_device()
verbose = True
# Initialize model
model = Darknet(cfg, img_size).to(device)
# Load weights
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(
torch.load(weights, map_location=device)['model'])
else: # darknet format
_ = load_darknet_weights(model, weights)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
else:
device = next(model.parameters()).device # get model device
verbose = False
# Configure run
data = parse_data_cfg(data)
nc = int(data['classes']) # number of classes
test_path = data['valid'] # path to test images
names = load_classes(data['names']) # class names
# Dataloader
dataset = LoadImagesAndLabels(test_path, img_size, batch_size)
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=min(os.cpu_count(), batch_size),
pin_memory=True,
collate_fn=dataset.collate_fn)
seen = 0
model.eval()
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP',
'F1')
p, r, f1, mp, mr, map, mf1 = 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (imgs, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=s)):
targets = targets.to(device)
imgs = imgs.to(device)
_, _, height, width = imgs.shape # batch size, channels, height, width
# Plot images with bounding boxes
if batch_i == 0 and not os.path.exists('test_batch0.jpg'):
plot_images(imgs=imgs,
targets=targets,
paths=paths,
fname='test_batch0.jpg')
# Run model
inf_out, train_out = model(imgs) # inference and training outputs
# Compute loss
if hasattr(model, 'hyp'): # if model has loss hyperparameters
loss += compute_loss(train_out, targets,
model)[1][:3].cpu() # GIoU, obj, cls
# Run NMS
output = non_max_suppression(inf_out,
conf_thres=conf_thres,
nms_thres=nms_thres)
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append(([], torch.Tensor(), torch.Tensor(), tcls))
continue
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
scale_coords(imgs[si].shape[1:], box,
shapes[si]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for di, d in enumerate(pred):
jdict.append({
'image_id': image_id,
'category_id': coco91class[int(d[6])],
'bbox': [floatn(x, 3) for x in box[di]],
'score': floatn(d[4], 5)
})
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Assign all predictions as incorrect
correct = [0] * len(pred)
if nl:
detected = []
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5])
tbox[:, [0, 2]] *= width
tbox[:, [1, 3]] *= height
# Search for correct predictions
for i, (*pbox, pconf, pcls_conf, pcls) in enumerate(pred):
# Break if all targets already located in image
if len(detected) == nl:
break
# Continue if predicted class not among image classes
if pcls.item() not in tcls:
continue
# Best iou, index between pred and targets
m = (pcls == tcls_tensor).nonzero().view(-1)
iou, bi = bbox_iou(pbox, tbox[m]).max(0)
# If iou > threshold and class is correct mark as correct
if iou > iou_thres and m[
bi] not in detected: # and pcls == tcls[bi]:
correct[i] = 1
detected.append(m[bi])
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct, pred[:, 4].cpu(), pred[:, 6].cpu(), tcls))
# Compute statistics
stats = [np.concatenate(x, 0) for x in list(zip(*stats))] # to numpy
if len(stats):
p, r, ap, f1, ap_class = ap_per_class(*stats)
mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))
# Save JSON
if save_json and map and len(jdict):
imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataset.img_files]
with open('results.json', 'w') as file:
json.dump(jdict, file)
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
cocoGt = COCO('../coco/annotations/instances_val2014.json'
) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map = cocoEval.stats[1] # update mAP to pycocotools mAP
# Return results
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map, mf1, *(loss / len(dataloader)).tolist()), maps
float(ws[k]),
float(hs[k])],
'score':
float(scores[k])
} for k in range(det.shape[0])]
result = sorted(result, key=lambda x: x['score'])[-100:]
coco_result += result
t5_s = time.time()
print("convert to coco format uses: %.1f" % (t5_s - t3_s))
import json
json.dump(coco_result,
open(
"experiments/{}/{}_proposal_result.json".format(
pGen.name, pDataset.image_set[0]), "w"),
sort_keys=True,
indent=2)
coco_dt = coco.loadRes(coco_result)
coco_eval = COCOeval(coco, coco_dt)
coco_eval.params.iouType = "bbox"
coco_eval.params.maxDets = [1, 10, 100] # [100, 300, 1000]
coco_eval.params.useCats = False
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
t6_s = time.time()
print("coco eval uses: %.1f" % (t6_s - t5_s))
def infer(model, path, detections_file, resize, max_size, batch_size, mixed_precision=False, is_master=True, world=0, annotations=None, use_dali=True, is_validation=False, verbose=True):
'Run inference on images from path'
print('model',model)
backend = 'pytorch' if isinstance(model, Model) or isinstance(model, DDP) else 'tensorrt'
#print("backend",backend)
stride = model.module.stride if isinstance(model, DDP) else model.stride
#print('!!!!!!!!model.stride:', model.stride)
# Create annotations if none was provided
if not annotations:
annotations = tempfile.mktemp('.json')
images = [{ 'id': i, 'file_name': f} for i, f in enumerate(os.listdir(path))]
json.dump({ 'images': images }, open(annotations, 'w'))
# TensorRT only supports fixed input sizes, so override input size accordingly
if backend == 'tensorrt': max_size = max(model.input_size)
# Prepare dataset
if verbose: print('Preparing dataset...')
data_iterator = (DaliDataIterator if use_dali else DataIterator)(
path, resize, max_size, batch_size, stride,
world, annotations, training=False)
if verbose: print(data_iterator)
# Prepare model
if backend is 'pytorch':
# If we are doing validation during training,
# no need to register model with AMP again
if not is_validation:
if torch.cuda.is_available(): model = model.cuda()
model = amp.initialize(model, None,
opt_level = 'O2' if mixed_precision else 'O0',
keep_batchnorm_fp32 = True,
verbosity = 0)
model.eval()
if verbose:
print(' backend: {}'.format(backend))
print(' device: {} {}'.format(
world, 'cpu' if not torch.cuda.is_available() else 'gpu' if world == 1 else 'gpus'))
print(' batch: {}, precision: {}'.format(batch_size,
'unknown' if backend is 'tensorrt' else 'mixed' if mixed_precision else 'full'))
print('Running inference...')
results = []
profiler = Profiler(['infer', 'fw'])
with torch.no_grad():
for i, (data, ids, ratios) in enumerate(data_iterator):
# Forward pass
#print('start profiler')
profiler.start('fw')
#print("data:",data)
scores, boxes, classes = model(data)
profiler.stop('fw')
#cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2)
results.append([scores, boxes, classes, ids, ratios])
profiler.bump('infer')
if verbose and (profiler.totals['infer'] > 60 or i == len(data_iterator) - 1):
size = len(data_iterator.ids)
msg = '[{:{len}}/{}]'.format(min((i + 1) * batch_size,
size), size, len=len(str(size)))
msg += ' {:.3f}s/{}-batch'.format(profiler.means['infer'], batch_size)
msg += ' (fw: {:.3f}s)'.format(profiler.means['fw'])
msg += ', {:.1f} im/s'.format(batch_size / profiler.means['infer'])
print(msg, flush=True)
profiler.reset()
# Gather results from all devices
if verbose: print('Gathering results...')
results = [torch.cat(r, dim=0) for r in zip(*results)]
if world > 1:
for r, result in enumerate(results):
all_result = [torch.ones_like(result, device=result.device) for _ in range(world)]
torch.distributed.all_gather(list(all_result), result)
results[r] = torch.cat(all_result, dim=0)
if is_master:
# Copy buffers back to host
results = [r.cpu() for r in results]
# Collect detections
detections = []
processed_ids = set()
for scores, boxes, classes, image_id, ratios in zip(*results):
image_id = image_id.item()
if image_id in processed_ids:
continue
processed_ids.add(image_id)
keep = (scores > 0).nonzero()
scores = scores[keep].view(-1)
boxes = boxes[keep, :].view(-1, 4) / ratios
classes = classes[keep].view(-1).int()
#print('classes', classes)
for score, box, cat in zip(scores, boxes, classes):
x1, y1, x2, y2 = box.data.tolist()
cat = cat.item()
if 'annotations' in data_iterator.coco.dataset:
cat = data_iterator.coco.getCatIds()[cat]
#if cat !=3:
#continue
#print('cat',cat)
detections.append({
'image_id': image_id,
'score': score.item(),
'bbox': [x1, y1, x2 - x1 + 1, y2 - y1 + 1],
'category_id': cat
})
#show_detections(detections)
if detections:
# Save detections
if detections_file and verbose: print('Writing {}...'.format(detections_file))
detections = { 'annotations': detections }
detections['images'] = data_iterator.coco.dataset['images']
if 'categories' in data_iterator.coco.dataset:
detections['categories'] = [data_iterator.coco.dataset['categories']]
if detections_file:
json.dump(detections, open(detections_file, 'w'), indent=4)
# Evaluate model on dataset
if 'annotations' in data_iterator.coco.dataset:
if verbose: print('Evaluating model...')
with redirect_stdout(None):
coco_pred = data_iterator.coco.loadRes(detections['annotations'])
coco_eval = COCOeval(data_iterator.coco, coco_pred, 'bbox')
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
else:
print('No detections!')
def evaluate_coco(model,
dataset,
coco,
eval_type="bbox",
limit=0,
image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
# Pick TACO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
# Get corresponding TACO image IDs.
taco_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
# Load image
image = dataset.load_image(image_id)
# Run detection
t = time.time()
r = model.detect([image], verbose=0)[0]
# r = utils.fuse_instances(r)
t_prediction += (time.time() - t)
if not model.config.DETECTION_SCORE_RATIO:
scores = r["scores"]
else:
scores = r["scores"] / (r["full_scores"][:, 0] + 0.0001)
# Convert results to COCO format
# Cast masks to uint8 because COCO tools errors out on bool
image_results = build_coco_results(dataset, taco_image_ids[i:i + 1],
r["rois"], r["class_ids"], scores,
r["masks"].astype(np.uint8))
results.extend(image_results)
# Load results. This modifies results with additional attributes.
coco_results = coco.loadRes(results)
# utils.compute_confusion_matrix(coco_results, coco)
# Evaluate
cocoEval = COCOeval(coco, coco_results, eval_type)
cocoEval.params.imgIds = taco_image_ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
def compute_map(labels_and_predictions,
coco_gt,
use_cpp_extension=True,
nms_on_tpu=True):
"""Use model predictions to compute mAP.
The evaluation code is largely copied from the MLPerf reference
implementation. While it is possible to write the evaluation as a tensor
metric and use Estimator.evaluate(), this approach was selected for simplicity
and ease of duck testing.
Args:
labels_and_predictions: A map from TPU predict method.
coco_gt: ground truch COCO object.
use_cpp_extension: use cocoeval C++ library.
nms_on_tpu: do NMS on TPU.
Returns:
Evaluation result.
"""
predictions = []
tic = time.time()
if nms_on_tpu:
p = []
for i in labels_and_predictions:
for j in i:
p.append(np.array(j, dtype=np.float32))
predictions = np.concatenate(list(p)).reshape((-1, 7))
else:
for example in labels_and_predictions:
if ssd_constants.IS_PADDED in example and example[
ssd_constants.IS_PADDED]:
continue
htot, wtot, _ = example[ssd_constants.RAW_SHAPE]
pred_box = example['pred_box']
pred_scores = example['pred_scores']
indices = example['indices']
loc, label, prob = decode_single(pred_box, pred_scores, indices,
ssd_constants.OVERLAP_CRITERIA,
ssd_constants.MAX_NUM_EVAL_BOXES,
ssd_constants.MAX_NUM_EVAL_BOXES)
for loc_, label_, prob_ in zip(loc, label, prob):
# Ordering convention differs, hence [1], [0] rather than [0], [1]
predictions.append([
int(example[ssd_constants.SOURCE_ID]), loc_[1] * wtot,
loc_[0] * htot, (loc_[3] - loc_[1]) * wtot,
(loc_[2] - loc_[0]) * htot, prob_,
ssd_constants.CLASS_INV_MAP[label_]
])
toc = time.time()
tf.logging.info('Prepare predictions DONE (t={:0.2f}s).'.format(toc - tic))
if coco_gt is None:
coco_gt = create_coco(FLAGS.val_json_file,
use_cpp_extension=use_cpp_extension)
if use_cpp_extension:
coco_dt = coco_gt.LoadRes(np.array(predictions, dtype=np.float32))
# copybara:strip_begin
coco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iou_type='bbox')
# copybara:strip_end
# copybara:insert coco_eval = COCOeval(coco_gt, coco_dt, iou_type='bbox')
coco_eval.Evaluate()
coco_eval.Accumulate()
coco_eval.Summarize()
stats = coco_eval.GetStats()
else:
coco_dt = coco_gt.loadRes(np.array(predictions))
coco_eval = COCOeval(coco_gt, coco_dt, iouType='bbox')
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats = coco_eval.stats
print('Current AP: {:.5f}'.format(stats[0]))
metric_names = [
'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'ARmax1', 'ARmax10',
'ARmax100', 'ARs', 'ARm', 'ARl'
]
coco_time = time.time()
tf.logging.info('COCO eval DONE (t={:0.2f}s).'.format(coco_time - toc))
# Prefix with "COCO" to group in TensorBoard.
return {'COCO/' + key: value for key, value in zip(metric_names, stats)}
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
classwise=False,
proposal_nums=(100, 300, 1000),
iou_thrs=None,
metric_items=None):
"""Evaluation in COCO protocol.
Args:
results (list[list | tuple]): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated. Options are
'bbox', 'segm', 'proposal', 'proposal_fast'.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
classwise (bool): Whether to evaluating the AP for each class.
proposal_nums (Sequence[int]): Proposal number used for evaluating
recalls, such as recall@100, recall@1000.
Default: (100, 300, 1000).
iou_thrs (Sequence[float], optional): IoU threshold used for
evaluating recalls/mAPs. If set to a list, the average of all
IoUs will also be computed. If not specified, [0.50, 0.55,
0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] will be used.
Default: None.
metric_items (list[str] | str, optional): Metric items that will
be returned. If not specified, ``['AR@100', 'AR@300',
'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ]`` will be
used when ``metric=='proposal'``, ``['mAP', 'mAP_50', 'mAP_75',
'mAP_s', 'mAP_m', 'mAP_l']`` will be used when
``metric=='bbox' or metric=='segm'``.
Returns:
dict[str, float]: COCO style evaluation metric.
"""
metrics = metric if isinstance(metric, list) else [metric]
allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast']
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError(f'metric {metric} is not supported')
if iou_thrs is None:
iou_thrs = np.linspace(.5,
0.95,
int(np.round((0.95 - .5) / .05)) + 1,
endpoint=True)
if metric_items is not None:
if not isinstance(metric_items, list):
metric_items = [metric_items]
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
eval_results = OrderedDict()
cocoGt = self.coco
for metric in metrics:
msg = f'Evaluating {metric}...'
if logger is None:
msg = '\n' + msg
print_log(msg, logger=logger)
if metric == 'proposal_fast':
ar = self.fast_eval_recall(results,
proposal_nums,
iou_thrs,
logger='silent')
log_msg = []
for i, num in enumerate(proposal_nums):
eval_results[f'AR@{num}'] = ar[i]
log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}')
log_msg = ''.join(log_msg)
print_log(log_msg, logger=logger)
continue
if metric not in result_files:
raise KeyError(f'{metric} is not in results')
try:
cocoDt = cocoGt.loadRes(result_files[metric])
except IndexError:
print_log('The testing results of the whole dataset is empty.',
logger=logger,
level=logging.ERROR)
break
iou_type = 'bbox' if metric == 'proposal' else metric
cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
cocoEval.params.catIds = self.cat_ids
cocoEval.params.imgIds = self.img_ids
cocoEval.params.maxDets = list(proposal_nums)
cocoEval.params.iouThrs = iou_thrs
# mapping of cocoEval.stats
coco_metric_names = {
'mAP': 0,
'mAP_50': 1,
'mAP_75': 2,
'mAP_s': 3,
'mAP_m': 4,
'mAP_l': 5,
'AR@100': 6,
'AR@300': 7,
'AR@1000': 8,
'AR_s@1000': 9,
'AR_m@1000': 10,
'AR_l@1000': 11
}
if metric_items is not None:
for metric_item in metric_items:
if metric_item not in coco_metric_names:
raise KeyError(
f'metric item {metric_item} is not supported')
if metric == 'proposal':
cocoEval.params.useCats = 0
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if metric_items is None:
metric_items = [
'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000',
'AR_m@1000', 'AR_l@1000'
]
for item in metric_items:
val = float(
f'{cocoEval.stats[coco_metric_names[item]]:.3f}')
eval_results[item] = val
else:
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if classwise: # Compute per-category AP
# Compute per-category AP
# from https://github.com/facebookresearch/detectron2/
precisions = cocoEval.eval['precision']
# precision: (iou, recall, cls, area range, max dets)
assert len(self.cat_ids) == precisions.shape[2]
results_per_category = []
for idx, catId in enumerate(self.cat_ids):
# area range index 0: all area ranges
# max dets index -1: typically 100 per image
nm = self.coco.loadCats(catId)[0]
precision = precisions[:, :, idx, 0, -1]
precision = precision[precision > -1]
if precision.size:
ap = np.mean(precision)
else:
ap = float('nan')
results_per_category.append(
(f'{nm["name"]}', f'{float(ap):0.3f}'))
num_columns = min(6, len(results_per_category) * 2)
results_flatten = list(
itertools.chain(*results_per_category))
headers = ['category', 'AP'] * (num_columns // 2)
results_2d = itertools.zip_longest(*[
results_flatten[i::num_columns]
for i in range(num_columns)
])
table_data = [headers]
table_data += [result for result in results_2d]
table = AsciiTable(table_data)
print_log('\n' + table.table, logger=logger)
if metric_items is None:
metric_items = [
'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l'
]
for metric_item in metric_items:
key = f'{metric}_{metric_item}'
val = float(
f'{cocoEval.stats[coco_metric_names[metric_item]]:.3f}'
)
eval_results[key] = val
ap = cocoEval.stats[:6]
eval_results[f'{metric}_mAP_copypaste'] = (
f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} '
f'{ap[4]:.3f} {ap[5]:.3f}')
if tmp_dir is not None:
tmp_dir.cleanup()
return eval_results
def test(
data,
weights=None,
batch_size=32,
imgsz=640,
conf_thres=0.001,
iou_thres=0.6, # for NMS
save_json=False,
single_cls=False,
augment=False,
verbose=False,
model=None,
dataloader=None,
save_dir=Path(''), # for saving images
save_txt=False, # for auto-labelling
save_hybrid=False, # for hybrid auto-labelling
save_conf=False, # save auto-label confidences
plots=True,
wandb_logger=None,
compute_loss=None,
half_precision=True,
is_coco=False):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
set_logging()
device = select_device(opt.device, batch_size=batch_size)
# Directories
save_dir = Path(
increment_path(Path(opt.project) / opt.name,
exist_ok=opt.exist_ok)) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(
parents=True, exist_ok=True) # make dir
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
gs = max(int(model.stride.max()), 32) # grid size (max stride)
imgsz = check_img_size(imgsz, s=gs) # check img_size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Half
half = device.type != 'cpu' and half_precision # half precision only supported on CUDA
if half:
model.half()
# Configure
model.eval()
if isinstance(data, str):
is_coco = data.endswith('coco.yaml')
with open(data) as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
check_dataset(data) # check
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Logging
log_imgs = 0
if wandb_logger and wandb_logger.wandb:
log_imgs = min(wandb_logger.log_imgs, 100)
# Dataloader
if not training:
if device.type != 'cpu':
model(
torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(
next(model.parameters()))) # run once
task = opt.task if opt.task in (
'train', 'val', 'test') else 'val' # path to train/val/test images
dataloader = create_dataloader(data[task],
imgsz,
batch_size,
gs,
opt,
pad=0.5,
rect=True,
prefix=colorstr(f'{task}: '))[0]
seen = 0
confusion_matrix = ConfusionMatrix(nc=nc)
names = {
k: v
for k, v in enumerate(
model.names if hasattr(model, 'names') else model.module.names)
}
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R',
'[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class, wandb_images = [], [], [], [], []
for batch_i, (img, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=s)):
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
with torch.no_grad():
# Run model
t = time_synchronized()
out, train_out = model(
img, augment=augment) # inference and training outputs
t0 += time_synchronized() - t
# Compute loss
if compute_loss:
loss += compute_loss([x.float() for x in train_out],
targets)[1][:3] # box, obj, cls
# Run NMS
targets[:, 2:] *= torch.Tensor([width, height, width,
height]).to(device) # to pixels
lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)
] if save_hybrid else [] # for autolabelling
t = time_synchronized()
out = non_max_suppression(out,
conf_thres=conf_thres,
iou_thres=iou_thres,
labels=lb,
multi_label=True)
t1 += time_synchronized() - t
# Statistics per image
for si, pred in enumerate(out):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
path = Path(paths[si])
seen += 1
if len(pred) == 0:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Predictions
predn = pred.clone()
scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0],
shapes[si][1]) # native-space pred
# Append to text file
if save_txt:
gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0
]] # normalization gain whwh
for *xyxy, conf, cls in predn.tolist():
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) /
gn).view(-1).tolist() # normalized xywh
line = (cls, *xywh,
conf) if save_conf else (cls,
*xywh) # label format
with open(save_dir / 'labels' / (path.stem + '.txt'),
'a') as f:
f.write(('%g ' * len(line)).rstrip() % line + '\n')
# W&B logging - Media Panel Plots
if len(
wandb_images
) < log_imgs and wandb_logger.current_epoch > 0: # Check for test operation
if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0:
box_data = [{
"position": {
"minX": xyxy[0],
"minY": xyxy[1],
"maxX": xyxy[2],
"maxY": xyxy[3]
},
"class_id": int(cls),
"box_caption": "%s %.3f" % (names[cls], conf),
"scores": {
"class_score": conf
},
"domain": "pixel"
} for *xyxy, conf, cls in pred.tolist()]
boxes = {
"predictions": {
"box_data": box_data,
"class_labels": names
}
} # inference-space
wandb_images.append(
wandb_logger.wandb.Image(img[si],
boxes=boxes,
caption=path.name))
wandb_logger.log_training_progress(
predn, path,
names) if wandb_logger and wandb_logger.wandb_run else None
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(
path.stem) if path.stem.isnumeric() else path.stem
box = xyxy2xywh(predn[:, :4]) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({
'image_id':
image_id,
'category_id':
coco91class[int(p[5])] if is_coco else int(p[5]),
'bbox': [round(x, 3) for x in b],
'score':
round(p[4], 5)
})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0],
niou,
dtype=torch.bool,
device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5])
scale_coords(img[si].shape[1:], tbox, shapes[si][0],
shapes[si][1]) # native-space labels
if plots:
confusion_matrix.process_batch(
predn, torch.cat((labels[:, 0:1], tbox), 1))
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(
-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(
-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(predn[pi, :4], tbox[ti]).max(
1) # best ious, indices
# Append detections
detected_set = set()
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d.item() not in detected_set:
detected_set.add(d.item())
detected.append(d)
correct[
pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(
detected
) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append(
(correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if plots and batch_i < 3:
f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels
Thread(target=plot_images,
args=(img, targets, paths, f, names),
daemon=True).start()
f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions
Thread(target=plot_images,
args=(img, output_to_target(out), paths, f, names),
daemon=True).start()
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats,
plot=plots,
save_dir=save_dir,
names=names)
ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%12i' * 2 + '%12.3g' * 4 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3
for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print(
'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g'
% t)
# Plots
if plots:
confusion_matrix.plot(save_dir=save_dir, names=list(names.values()))
if wandb_logger and wandb_logger.wandb:
val_batches = [
wandb_logger.wandb.Image(str(f), caption=f.name)
for f in sorted(save_dir.glob('test*.jpg'))
]
wandb_logger.log({"Validation": val_batches})
if wandb_images:
wandb_logger.log({"Bounding Box Debugger/Images": wandb_images})
# Save JSON
if save_json and len(jdict):
w = Path(weights[0] if isinstance(weights, list) else weights
).stem if weights is not None else '' # weights
anno_json = '../coco/annotations/instances_val2017.json' # annotations json
pred_json = str(save_dir / f"{w}_predictions.json") # predictions json
print('\nEvaluating pycocotools mAP... saving %s...' % pred_json)
with open(pred_json, 'w') as f:
json.dump(jdict, f)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
anno = COCO(anno_json) # init annotations api
pred = anno.loadRes(pred_json) # init predictions api
eval = COCOeval(anno, pred, 'bbox')
if is_coco:
eval.params.imgIds = [
int(Path(x).stem) for x in dataloader.dataset.img_files
] # image IDs to evaluate
eval.evaluate()
eval.accumulate()
eval.summarize()
map, map50 = eval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print(f'pycocotools unable to run: {e}')
# Return results
model.float() # for training
if not training:
s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
print(f"Results saved to {save_dir}{s}")
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(generator, model, anchors, json_path, imsize=448, threshold=0.5):
""" Use the pycocotools to evaluate a COCO model on a dataset.
Args
generator : The generator for generating the evaluation data.
model : The model to evaluate.
threshold : The score threshold to use.
"""
# start collecting results
import pickle
if os.path.exists('coco_eval_temp.pk'):
results, image_ids = pickle.load(open('coco_eval_temp.pk', 'rb'))
else:
results = []
image_ids = []
for index in range(generator.size()):
# if index % 50 == 0:
# print()
print(index, end='\r')
image = generator.load_image(index)
image, scale = resize_image(image, 360, imsize)
image = np.expand_dims(image, 0)
boxes = get_yolo_boxes(model,
image,
imsize, imsize,
anchors,
0.5,
0.5,
preprocess=True)[0]
boxes, scores, labels = boundbox2cocobox(boxes, scale)
# assert len(boxes) > 0
# compute predicted labels and scores
image_id = int(os.path.split(generator.instances[index]['filename'])[-1][:-4])
for box, score, label in zip(boxes, scores, labels):
# scores are sorted, so we can break
if score < threshold:
break
# append detection for each positively labeled class
image_result = {
'image_id': image_id,
'category_id': label_to_coco_label(label), # todo:
'score': float(score),
'bbox': box,
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(image_id)
with open('coco_eval_temp.pk', 'wb') as wr:
pickle.dump([results, image_ids], wr)
if not len(results):
return
import json
# write output
json.dump(results, open('{}_bbox_results.json'.format('val2017'), 'w'), indent=4)
json.dump(image_ids, open('{}_processed_image_ids.json'.format('val2017'), 'w'), indent=4)
# load results in COCO evaluation tool
coco_true = COCO(json_path)
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format('val2017'))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval.stats
def run(
data,
weights=None, # model.pt path(s)
batch_size=32, # batch size
imgsz=640, # inference size (pixels)
conf_thres=0.001, # confidence threshold
iou_thres=0.6, # NMS IoU threshold
task='val', # train, val, test, speed or study
device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
workers=8, # max dataloader workers (per RANK in DDP mode)
single_cls=False, # treat as single-class dataset
augment=False, # augmented inference
verbose=False, # verbose output
save_txt=False, # save results to *.txt
save_hybrid=False, # save label+prediction hybrid results to *.txt
save_conf=False, # save confidences in --save-txt labels
save_json=False, # save a COCO-JSON results file
project=ROOT / 'runs/val', # save to project/name
name='exp', # save to project/name
exist_ok=False, # existing project/name ok, do not increment
half=True, # use FP16 half-precision inference
dnn=False, # use OpenCV DNN for ONNX inference
model=None,
dataloader=None,
save_dir=Path(''),
plots=True,
callbacks=Callbacks(),
compute_loss=None,
):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device, pt, jit, engine = next(model.parameters(
)).device, True, False, False # get model device, PyTorch model
half &= device.type != 'cpu' # half precision only supported on CUDA
model.half() if half else model.float()
else: # called directly
device = select_device(device, batch_size=batch_size)
# Directories
save_dir = increment_path(Path(project) / name,
exist_ok=exist_ok) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(
parents=True, exist_ok=True) # make dir
# Load model
model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data)
stride, pt, jit, onnx, engine = model.stride, model.pt, model.jit, model.onnx, model.engine
imgsz = check_img_size(imgsz, s=stride) # check image size
half &= (
pt or jit or onnx or engine
) and device.type != 'cpu' # FP16 supported on limited backends with CUDA
if pt or jit:
model.model.half() if half else model.model.float()
elif engine:
batch_size = model.batch_size
if model.trt_fp16_input != half:
LOGGER.info('model ' +
('requires' if model.
trt_fp16_input else 'incompatible with') +
' --half. Adjusting automatically.')
half = model.trt_fp16_input
else:
half = False
batch_size = 1 # export.py models default to batch-size 1
device = torch.device('cpu')
LOGGER.info(
f'Forcing --batch-size 1 square inference shape(1,3,{imgsz},{imgsz}) for non-PyTorch backends'
)
# Data
data = check_dataset(data) # check
# Configure
model.eval()
is_coco = isinstance(data.get('val'), str) and data['val'].endswith(
'coco/val2017.txt') # COCO dataset
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if not training:
model.warmup(imgsz=(1 if pt else batch_size, 3, imgsz, imgsz),
half=half) # warmup
pad = 0.0 if task in ('speed', 'benchmark') else 0.5
rect = False if task == 'benchmark' else pt # square inference for benchmarks
task = task if task in (
'train', 'val', 'test') else 'val' # path to train/val/test images
dataloader = create_dataloader(data[task],
imgsz,
batch_size,
stride,
single_cls,
pad=pad,
rect=rect,
workers=workers,
prefix=colorstr(f'{task}: '))[0]
seen = 0
confusion_matrix = ConfusionMatrix(nc=nc)
names = {
k: v
for k, v in enumerate(
model.names if hasattr(model, 'names') else model.module.names)
}
class_map = coco80_to_coco91_class() if is_coco else list(range(1000))
s = ('%20s' + '%11s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R',
'[email protected]', '[email protected]:.95')
dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0,
0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
pbar = tqdm(dataloader,
desc=s,
bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}') # progress bar
for batch_i, (im, targets, paths, shapes) in enumerate(pbar):
t1 = time_sync()
if pt or jit or engine:
im = im.to(device, non_blocking=True)
targets = targets.to(device)
im = im.half() if half else im.float() # uint8 to fp16/32
im /= 255 # 0 - 255 to 0.0 - 1.0
nb, _, height, width = im.shape # batch size, channels, height, width
t2 = time_sync()
dt[0] += t2 - t1
# Inference
out, train_out = model(im) if training else model(
im, augment=augment, val=True) # inference, loss outputs
dt[1] += time_sync() - t2
# Loss
if compute_loss:
loss += compute_loss([x.float() for x in train_out],
targets)[1] # box, obj, cls
# NMS
targets[:, 2:] *= torch.Tensor([width, height, width,
height]).to(device) # to pixels
lb = [targets[targets[:, 0] == i, 1:]
for i in range(nb)] if save_hybrid else [] # for autolabelling
t3 = time_sync()
out = non_max_suppression(out,
conf_thres,
iou_thres,
labels=lb,
multi_label=True,
agnostic=single_cls)
dt[2] += time_sync() - t3
# Metrics
for si, pred in enumerate(out):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
path, shape = Path(paths[si]), shapes[si][0]
seen += 1
if len(pred) == 0:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Predictions
if single_cls:
pred[:, 5] = 0
predn = pred.clone()
scale_coords(im[si].shape[1:], predn[:, :4], shape,
shapes[si][1]) # native-space pred
# Evaluate
if nl:
tbox = xywh2xyxy(labels[:, 1:5]) # target boxes
scale_coords(im[si].shape[1:], tbox, shape,
shapes[si][1]) # native-space labels
labelsn = torch.cat((labels[:, 0:1], tbox),
1) # native-space labels
correct = process_batch(predn, labelsn, iouv)
if plots:
confusion_matrix.process_batch(predn, labelsn)
else:
correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool)
stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(),
tcls)) # (correct, conf, pcls, tcls)
# Save/log
if save_txt:
save_one_txt(predn,
save_conf,
shape,
file=save_dir / 'labels' / (path.stem + '.txt'))
if save_json:
save_one_json(predn, jdict, path,
class_map) # append to COCO-JSON dictionary
callbacks.run('on_val_image_end', pred, predn, path, names, im[si])
# Plot images
if plots and batch_i < 3:
f = save_dir / f'val_batch{batch_i}_labels.jpg' # labels
Thread(target=plot_images,
args=(im, targets, paths, f, names),
daemon=True).start()
f = save_dir / f'val_batch{batch_i}_pred.jpg' # predictions
Thread(target=plot_images,
args=(im, output_to_target(out), paths, f, names),
daemon=True).start()
# Compute metrics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats,
plot=plots,
save_dir=save_dir,
names=names)
ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%11i' * 2 + '%11.3g' * 4 # print format
LOGGER.info(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
LOGGER.info(pf %
(names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3 for x in dt) # speeds per image
if not training:
shape = (batch_size, 3, imgsz, imgsz)
LOGGER.info(
f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}'
% t)
# Plots
if plots:
confusion_matrix.plot(save_dir=save_dir, names=list(names.values()))
callbacks.run('on_val_end')
# Save JSON
if save_json and len(jdict):
w = Path(weights[0] if isinstance(weights, list) else weights
).stem if weights is not None else '' # weights
anno_json = str(
Path(data.get('path', '../coco')) /
'annotations/instances_val2017.json') # annotations json
pred_json = str(save_dir / f"{w}_predictions.json") # predictions json
LOGGER.info(f'\nEvaluating pycocotools mAP... saving {pred_json}...')
with open(pred_json, 'w') as f:
json.dump(jdict, f)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
check_requirements(['pycocotools'])
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
anno = COCO(anno_json) # init annotations api
pred = anno.loadRes(pred_json) # init predictions api
eval = COCOeval(anno, pred, 'bbox')
if is_coco:
eval.params.imgIds = [
int(Path(x).stem) for x in dataloader.dataset.im_files
] # image IDs to evaluate
eval.evaluate()
eval.accumulate()
eval.summarize()
map, map50 = eval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
LOGGER.info(f'pycocotools unable to run: {e}')
# Return results
model.float() # for training
if not training:
s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}")
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate(self, model):
"""
COCO average precision (AP) Evaluation. Iterate inference on the test dataset
and the results are evaluated by COCO API.
Args:
model : model object
Returns:
ap50_95 (float) : calculated COCO AP for IoU=50:95
ap50 (float) : calculated COCO AP for IoU=50
"""
model.eval()
ids = []
data_dict = []
num_images = len(self.dataset)
print('total number of images: %d' % (num_images))
# start testing
for index in range(num_images): # all the data in val2017
if index % 500 == 0:
print('[Eval: %d / %d]' % (index, num_images))
# load an image
img, id_ = self.dataset.pull_image(index)
h, w, _ = img.shape
size = np.array([[w, h, w, h]])
# preprocess
x, _, _, scale, offset = self.transform(img)
x = x.unsqueeze(0).to(self.device)
id_ = int(id_)
ids.append(id_)
# inference
with torch.no_grad():
outputs = model(x)
bboxes, scores, cls_inds = outputs
# map the boxes to original image
bboxes -= offset
bboxes /= scale
bboxes *= size
for i, box in enumerate(bboxes):
x1 = float(box[0])
y1 = float(box[1])
x2 = float(box[2])
y2 = float(box[3])
label = self.dataset.class_ids[int(cls_inds[i])]
bbox = [x1, y1, x2 - x1, y2 - y1]
score = float(scores[i]) # object score * class score
A = {
"image_id": id_,
"category_id": label,
"bbox": bbox,
"score": score
} # COCO json format
data_dict.append(A)
annType = ['segm', 'bbox', 'keypoints']
# Evaluate the Dt (detection) json comparing with the ground truth
if len(data_dict) > 0:
print('evaluating ......')
cocoGt = self.dataset.coco
# workaround: temporarily write data to json file because pycocotools can't process dict in py36.
if self.testset:
json.dump(data_dict, open('coco_test-dev.json', 'w'))
cocoDt = cocoGt.loadRes('coco_test-dev.json')
return -1, -1
else:
_, tmp = tempfile.mkstemp()
json.dump(data_dict, open(tmp, 'w'))
cocoDt = cocoGt.loadRes(tmp)
cocoEval = COCOeval(self.dataset.coco, cocoDt, annType[1])
cocoEval.params.imgIds = ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
ap50_95, ap50 = cocoEval.stats[0], cocoEval.stats[1]
print('ap50_95 : ', ap50_95)
print('ap50 : ', ap50)
self.map = ap50_95
self.ap50_95 = ap50_95
self.ap50 = ap50
return ap50, ap50_95
else:
return 0, 0
def evaluate(config_file="configs/COCO-Detection/yolov5-small.yaml",
batch_size=16,
data="data/coco2017.yaml",
image_size=640,
weights=None,
confidence_thresholds=0.001,
iou_thresholds=0.6,
save_json=False,
merge=False,
augment=False,
verbose=False,
save_txt=False,
model=None,
dataloader=None):
with open(data) as f:
data_dict = yaml.load(f, Loader=yaml.FullLoader)
number_classes, names = int(
data_dict["number_classes"]), data_dict["names"]
assert len(
names
) == number_classes, f"{len(names)} names found for nc={number_classes} dataset in {data}"
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
device = select_device(args.device, batch_size=args.batch_size)
if save_txt:
if os.path.exists("outputs"):
shutil.rmtree("outputs") # delete output folder
os.makedirs("outputs") # make new output folder
# Create model
model = YOLO(config_file=config_file,
number_classes=number_classes).to(device)
# Load model
model.load_state_dict(torch.load(weights)["state_dict"])
model.float()
model.fuse()
model.eval()
# Half
half = device.type != "cpu" # half precision only supported on CUDA
if half:
model.half()
# Configure
model.eval()
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if not training:
image = torch.zeros((1, 3, image_size, image_size),
device=device) # init image
_ = model(image.half() if half else image
) if device.type != "cpu" else None # run once
dataroot = data_dict["test"] if data_dict["test"] else data_dict[
"val"] # path to val/test images
dataset, dataloader = create_dataloader(dataroot=dataroot,
image_size=image_size,
batch_size=batch_size,
hyper_parameters=None,
augment=False,
cache=False,
rect=True)
seen = 0
coco91class = coco80_to_coco91_class()
context = f"{'Class':>20}{'Images':>12}{'Targets':>12}{'P':>12}{'R':>12}{'[email protected]':>12}{'[email protected]:.95':>12}"
p, r, f1, mp, mr, map50, map, inference_time, nms_time = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for _, (image, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=context)):
image = image.to(device, non_blocking=True)
image = image.half() if half else image.float() # uint8 to fp16/32
image /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = image.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Disable gradients
with torch.no_grad():
# Run model
t = time_synchronized()
prediction, outputs = model(
image, augment=augment) # inference and training outputs
inference_time += time_synchronized() - t
# Compute loss
if training: # if model has loss hyper parameters
loss += compute_loss([x.float() for x in outputs], targets,
model)[1][:3] # GIoU, obj, cls
# Run NMS
t = time_synchronized()
prediction = non_max_suppression(
prediction=prediction,
confidence_thresholds=confidence_thresholds,
iou_thresholds=iou_thresholds,
merge=merge,
classes=None,
agnostic=False)
nms_time += time_synchronized() - t
# Statistics per image
for si, pred in enumerate(prediction):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
if save_txt:
gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0
]] # normalization gain whwh
txt_path = os.path.join("outputs",
paths[si].split("/")[-1][:-4])
pred[:, :4] = scale_coords(image[si].shape[1:], pred[:, :4],
shapes[si][0],
shapes[si][1]) # to original
for *xyxy, conf, cls in pred:
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) /
gn).view(-1).tolist() # normalized xywh
with open(txt_path + ".txt", "a") as f:
f.write(
("%g " * 5 + "\n") % (cls, *xywh)) # label format
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = si[:-4]
box = pred[:, :4].clone() # xyxy
scale_coords(image[si].shape[1:], box, shapes[si][0],
shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({
"image_id":
int(image_id) if image_id.isnumeric() else image_id,
"category_id":
coco91class[int(p[5])],
"bbox": [round(x, 3) for x in b],
"score":
round(p[4], 5)
})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0],
niou,
dtype=torch.bool,
device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(
-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(
-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(pred[pi, :4], tbox[ti]).max(
1) # best ious, indices
# Append detections
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d not in detected:
detected.append(d)
correct[
pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(
detected
) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append(
(correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats)
p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(
1) # [P, R, [email protected], [email protected]:0.95]
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(
np.int64), minlength=number_classes) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
print(
f"{'all':>20}{seen:>12}{nt.sum():>12}{mp:>12.3f}{mr:>12.3f}{map50:>12.3f}{map:>12.3f}"
)
# Print results per class
if verbose:
for i, c in enumerate(ap_class):
print(
f"{names[c]:>20}{seen:>12}{nt[c]:>12}{p[i]:>12.3f}{r[i]:>12.3f}{ap50[i]:>12.3f}{ap[i]:>12.3f}"
)
# Print speeds
if not training:
print(
"Speed: "
f"{inference_time / seen * 1000:.1f}/"
f"{nms_time / seen * 1000:.1f}/"
f"{(inference_time + nms_time) / seen * 1000:.1f} ms "
f"inference/NMS/total per {image_size}x{image_size} image at batch-size {batch_size}"
)
# Save JSON
if save_json and len(jdict):
f = f"detections_val2017_{weights.split('/')[-1].replace('.pth', '')}_results.json"
print(f"\nCOCO mAP with pycocotools... saving {f}...")
with open(f, "w") as file:
json.dump(jdict, file)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
imgIds = [
int(x.split("/")[-1][:-4])
for x in dataloader.dataset.image_files
]
cocoGt = COCO(
glob.glob("data/coco2017/annotations/instances_val*.json")[0])
cocoDt = cocoGt.loadRes(f) # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, "bbox")
cocoEval.params.imgIds = imgIds # image IDs to evaluate
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map, map50 = cocoEval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print(f"ERROR: pycocotools unable to run: {e}")
# Return results
model.float() # for training
maps = np.zeros(int(data_dict["number_classes"])) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps
def test(
data,
weights=None,
batch_size=16,
imgsz=640,
conf_thres=0.001,
iou_thres=0.6, # for NMS
save_json=False,
verbose=False,
model=None,
dataloader=None,
logdir='./runs',
merge=False):
# Initialize/load model and set device
if model is None:
training = False
device = torch_utils.select_device(opt.device, batch_size=batch_size)
# Remove previous
for f in glob.glob(os.path.join(logdir, 'test_batch*.jpg')):
os.remove(f)
# Load model
model = torch.load(
weights, map_location=device)['model'].float() # load to FP32
torch_utils.model_info(model)
model.fuse()
model.to(device)
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
else: # called by train.py
training = True
device = next(model.parameters()).device # get model device
# Half
half = device.type != 'cpu' and torch.cuda.device_count(
) == 1 # half precision only supported on single-GPU
half = False
if half:
model.half() # to FP16
# Configure
model.eval()
with open(data) as f:
data = yaml.load(f, Loader=yaml.FullLoader) # model dict
nc = int(data['num_classes']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
losser = YoloLoss(model)
# Dataloader
if dataloader is None: # not training
merge = opt.merge # use Merge NMS
img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img
_ = model(img.half() if half else img
) if device.type != 'cpu' else None # run once
path = data['test'] if opt.task == 'test' else data[
'val'] # path to val/test images
dataloader = kitti.create_dataloader(path,
imgsz,
batch_size,
int(max(model.stride)),
config=None,
augment=False,
cache=False,
pad=0.5,
rect=True)[0]
seen = 0
names = data['names']
kitti8class = data_utils.kitti8_classes()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R',
'[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (img, targets, paths,
shapes) in enumerate(tqdm.tqdm(dataloader, desc=s)):
targets.delete_by_mask()
targets.to_float32()
targ = ParamList(targets.size, True)
targ.copy_from(targets)
img_id = targets.get_field('img_id')
classes = targets.get_field('class')
bboxes = targets.get_field('bbox')
targets = torch.cat(
[img_id.unsqueeze(-1),
classes.unsqueeze(-1), bboxes], dim=-1)
img = img.to(device)
img = img.half() if half else img.float() # uint8 to fp16/32
# img /= 1.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Disable gradients
with torch.no_grad():
# Run model
t = torch_utils.time_synchronized()
inf_out, train_out = model(img) # inference and training outputs
t0 += torch_utils.time_synchronized() - t
# Compute loss
if training: # if model has loss hyperparameters
# loss += calc_loss([x.float() for x in train_out], targets, model)[1][:3] # GIoU, obj, cls
loss += losser([x.float() for x in train_out], targ)[1][:3]
# Run NMS
t = torch_utils.time_synchronized()
output = postprocess.apply_nms(inf_out,
nc,
conf_thres=conf_thres,
iou_thres=iou_thres,
merge=merge)
t1 += torch_utils.time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
# with open('test.txt', 'a') as file:
# [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred]
# Clip boxes to image bounds
utils.clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
utils.scale_coords(img[si].shape[1:], box, shapes[si][0],
shapes[si][1]) # to original shape
box = data_utils.xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({
'image_id': image_id,
'category_id': kitti8class[int(p[5])],
'bbox': [round(x, 3) for x in b],
'score': round(p[4], 5)
})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0],
niou,
dtype=torch.bool,
device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = data_utils.xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(
-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(
-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = metrics_utils.box_iou(
pred[pi, :4],
tbox[ti]).max(1) # best ious, indices
# Append detections
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d not in detected:
detected.append(d)
correct[
pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(
detected
) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append(
(correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if batch_i < 1:
f = os.path.join(logdir,
'test_batch%g_gt.jpg' % batch_i) # filename
visual_utils.plot_images(img, targets, paths, f,
names) # ground truth
f = os.path.join(logdir, 'test_batch%g_pred.jpg' % batch_i)
visual_utils.plot_images(img,
utils.output_to_target(
output, width, height), paths, f,
names) # predictions
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats):
p, r, ap, f1, ap_class = metrics_utils.ap_per_class(*stats)
p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(
1) # [P, R, [email protected], [email protected]:0.95]
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%12.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3
for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print(
'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g'
% t)
# Save JSON
if save_json and map50 and len(jdict):
imgIds = [
int(Path(x).stem.split('_')[-1])
for x in dataloader.dataset.img_files
]
f = 'detections_val2017_%s_results.json' % \
(weights.split(os.sep)[-1].replace('.pt', '') if weights else '') # filename
print('\nCOCO mAP with pycocotools... saving %s...' % f)
with open(f, 'w') as file:
json.dump(jdict, file)
try:
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
# https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
cocoGt = COCO(
glob.glob('../coco/annotations/instances_val*.json')
[0]) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes(f) # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # image IDs to evaluate
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map, map50 = cocoEval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except:
print(
'WARNING: pycocotools must be installed with numpy==1.17 to run correctly. '
'See https://github.com/cocodataset/cocoapi/issues/356')
# Return results
model.float() # for training
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(generator, model, threshold=0.05):
""" Use the pycocotools to evaluate a COCO model on a dataset.
Args
generator : The generator for generating the evaluation data.
model : The model to evaluate.
threshold : The score threshold to use.
"""
# start collecting results
results = []
image_ids = []
for index in progressbar.progressbar(range(generator.size()), prefix='COCO evaluation: '):
image = generator.load_image(index)
image = generator.preprocess_image(image)
image, scale = generator.resize_image(image)
if keras.backend.image_data_format() == 'channels_first':
image = image.transpose((2, 0, 1))
# run network
boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))
# correct boxes for image scale
boxes /= scale
# change to (x, y, w, h) (MS COCO standard)
boxes[:, :, 2] -= boxes[:, :, 0]
boxes[:, :, 3] -= boxes[:, :, 1]
# compute predicted labels and scores
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted, so we can break
if score < threshold:
break
# append detection for each positively labeled class
image_result = {
'image_id' : generator.image_ids[index],
'category_id' : generator.label_to_coco_label(label),
'score' : float(score),
'bbox' : box.tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(generator.image_ids[index])
if not len(results):
return
# write output
json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4)
json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4)
# load results in COCO evaluation tool
coco_true = generator.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval.stats
def main(random_seed, test_on_gt, only_test, overfit):
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
torch.cuda.manual_seed_all(random_seed)
n_epochs = 3
lr = 1e-2
wd = 0
lr_scheduler = True
train_db = JointCocoTasks()
network = JointClassifier()
optimizer = SGD(network.parameters(), lr=lr, weight_decay=wd)
experiment = JointClassifierExperiment(
network=network,
optimizer=optimizer,
dataset=train_db,
tensorboard=True,
seed=random_seed,
)
train_folder = "ablation-joint-classifier-seed:{s}".format(s=random_seed)
folder = os.path.join(SAVING_DIRECTORY, train_folder)
mkdir_p(folder)
if not only_test:
experiment.train_n_epochs(n_epochs,
overfit=overfit,
lr_scheduler=lr_scheduler)
torch.save(network.state_dict(), os.path.join(folder, "model.mdl"))
else:
network.load_state_dict(torch.load(os.path.join(folder, "model.mdl")))
for task_number in TASK_NUMBERS:
if test_on_gt:
test_db = CocoTasksTestGT(task_number)
else:
test_db = CocoTasksTest(task_number)
print("testing task {}".format(task_number), "---------------------")
# test_model
detections = experiment.do_test(test_db, task_number=task_number)
detections_file_name = "detections_tn:{}_tgt:{}.json".format(
task_number, test_on_gt)
# save detections
with open(os.path.join(folder, detections_file_name), "w") as f:
json.dump(detections, f)
# perform evaluation
with redirect_stdout(open(os.devnull, "w")):
gtCOCO = test_db.task_coco
dtCOCO = gtCOCO.loadRes(os.path.join(folder, detections_file_name))
cocoEval = COCOeval(gtCOCO, dtCOCO, "bbox")
cocoEval.params.catIds = 1
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("mAP:\t\t %1.6f" % cocoEval.stats[0])
print("[email protected]:\t\t %1.6f" % cocoEval.stats[1])
# save evaluation performance
result_file_name = "result_tn:{}_tgt:{}.txt".format(
task_number, test_on_gt)
with open(os.path.join(folder, result_file_name), "w") as f:
f.write("%1.6f, %1.6f" % (cocoEval.stats[0], cocoEval.stats[1]))
res_fn = '/home/admin/jupyter/wei/mmdetection-master/f_r50_v0.6_val_8.pkl.bbox.json'
ann_fn = '/home/admin/jupyter/wei/mmdetection-master/data/coco/annotations/instances_val2017_11cls.json'
root = 'jsons_from_v0.7/'
gt_path = root + 'gt_cls11map6_val2017.json'
res_path = root + 'res_cls11map6_v0.6.json'
gt_11cls = json.load(open(ann_fn, 'r'))
res_11cls = json.load(open(res_fn, 'r'))
for i in range(len(res_11cls)):
cat_id = res_11cls[i]['category_id']
res_11cls[i]['category_id'] = cat_id11map6[cat_id - 1]
json.dump(cls11map6(gt_11cls),
open(gt_path, 'w'),
ensure_ascii=False,
indent=2)
json.dump(res_11cls, open(res_path, 'w'), ensure_ascii=False, indent=2)
coco = COCO(gt_path)
coco_dets = coco.loadRes(res_path)
cocoEval = COCOeval(coco, coco_dets, 'bbox')
img_ids = coco.getImgIds()
cocoEval.params.imgIds = img_ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
table_classwise(cocoEval, coco)
def evaluate_workflow(coco_eval: COCOeval, cat_ids: List[int],
cat_names: List[str], out_dir: str) -> Dict[str, float]:
"""Execute evaluation."""
n_tit = 12 # number of evaluation titles
n_cls = len(cat_ids) # 10/8 classes for BDD100K detection/tracking
n_thr = 10 # [.5:.05:.95] T=10 IoU thresholds for evaluation
n_rec = 101 # [0:.01:1] R=101 recall thresholds for evaluation
n_area = 4 # A=4 object area ranges for evaluation
n_mdet = 3 # [1 10 100] M=3 thresholds on max detections per image
eval_param = {
"params": {
"imgIds": [],
"catIds": [],
"iouThrs":
np.linspace(
0.5,
0.95,
int(np.round((0.95 - 0.5) / 0.05) + 1),
endpoint=True,
).tolist(),
"recThrs":
np.linspace(
0.0,
1.00,
int(np.round((1.00 - 0.0) / 0.01) + 1),
endpoint=True,
).tolist(),
"maxDets": [1, 10, 100],
"areaRng": [
[0**2, 1e5**2],
[0**2, 32**2],
[32**2, 96**2],
[96**2, 1e5**2],
],
"useSegm":
0,
"useCats":
1,
},
"date":
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3],
"counts": [n_thr, n_rec, n_cls, n_area, n_mdet],
"precision": -np.ones(
(n_thr, n_rec, n_cls, n_area, n_mdet), order="F"),
"recall": -np.ones((n_thr, n_cls, n_area, n_mdet), order="F"),
}
stats_all = -np.ones((n_cls, n_tit))
for i, (cat_id, cat_name) in enumerate(zip(cat_ids, cat_names)):
print("\nEvaluate category: %s" % cat_name)
coco_eval.params.catIds = [cat_id]
# coco_eval.params.useSegm = ann_type == "segm"
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats_all[i, :] = coco_eval.stats
eval_param["precision"][:, :,
i, :, :] = coco_eval.eval["precision"].reshape(
(n_thr, n_rec, n_area, n_mdet))
eval_param["recall"][:, i, :, :] = coco_eval.eval["recall"].reshape(
(n_thr, n_area, n_mdet))
# Print evaluation results
stats = np.zeros((n_tit, 1))
print("\nOverall performance")
coco_eval.eval = eval_param
coco_eval.summarize()
for i in range(n_tit):
column = stats_all[:, i]
if len(column > -1) == 0:
stats[i] = -1
else:
stats[i] = np.mean(column[column > -1], axis=0)
score_titles = [
"AP",
"AP_50",
"AP_75",
"AP_small",
"AP_medium",
"AP_large",
"AR_max_1",
"AR_max_10",
"AR_max_100",
"AR_small",
"AR_medium",
"AR_large",
]
scores: Dict[str, float] = {}
for title, stat in zip(score_titles, stats):
scores[title] = stat.item()
if out_dir != "none":
write_eval(out_dir, scores, eval_param)
return scores
def test(data,
weights=None,
batch_size=16,
imgsz=640,
conf_thres=0.001,
iou_thres=0.6, # for NMS
save_json=False,
single_cls=False,
augment=False,
verbose=False,
model=None,
dataloader=None,
save_dir='',
merge=False,
save_txt=False):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
set_logging()
device = select_device(opt.device, batch_size=batch_size)
merge, save_txt = opt.merge, opt.save_txt # use Merge NMS, save *.txt labels
if save_txt:
out = Path('inference/output')
if os.path.exists(out):
shutil.rmtree(out) # delete output folder
os.makedirs(out) # make new output folder
# Remove previous
for f in glob.glob(str(Path(save_dir) / 'test_batch*.jpg')):
os.remove(f)
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Half
half = device.type != 'cpu' # half precision only supported on CUDA
if half:
model.half()
# Configure
model.eval()
with open(data) as f:
data = yaml.load(f, Loader=yaml.FullLoader) # model dict
check_dataset(data) # check
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if not training:
img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img
_ = model(img.half() if half else img) if device.type != 'cpu' else None # run once
path = data['test'] if opt.task == 'test' else data['val'] # path to val/test images
dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt,
hyp=None, augment=False, cache=False, pad=0.5, rect=True)[0]
seen = 0
names = model.names if hasattr(model, 'names') else model.module.names
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Disable gradients
with torch.no_grad():
# Run model
t = time_synchronized()
inf_out, train_out = model(img, augment=augment) # inference and training outputs
t0 += time_synchronized() - t
# Compute loss
if training: # if model has loss hyperparameters
loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # GIoU, obj, cls
# Run NMS
t = time_synchronized()
output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge)
t1 += time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
if save_txt:
gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh
txt_path = str(out / Path(paths[si]).stem)
pred[:, :4] = scale_coords(img[si].shape[1:], pred[:, :4], shapes[si][0], shapes[si][1]) # to original
for *xyxy, conf, cls in pred:
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh
with open(txt_path + '.txt', 'a') as f:
f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = Path(paths[si]).stem
box = pred[:, :4].clone() # xyxy
scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({'image_id': int(image_id) if image_id.isnumeric() else image_id,
'category_id': coco91class[int(p[5])],
'bbox': [round(x, 3) for x in b],
'score': round(p[4], 5)})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1) # best ious, indices
# Append detections
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d not in detected:
detected.append(d)
correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(detected) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if batch_i < 1:
f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i) # filename
plot_images(img, targets, paths, str(f), names) # ground truth
f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i)
plot_images(img, output_to_target(output, width, height), paths, str(f), names) # predictions
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats)
p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(1) # [P, R, [email protected], [email protected]:0.95]
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%12.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)
# Save JSON
if save_json and len(jdict):
f = 'detections_val2017_%s_results.json' % \
(weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '') # filename
print('\nCOCO mAP with pycocotools... saving %s...' % f)
with open(f, 'w') as file:
json.dump(jdict, file)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files]
cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes(f) # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # image IDs to evaluate
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map, map50 = cocoEval.stats[:2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print('ERROR: pycocotools unable to run: %s' % e)
# Return results
model.float() # for training
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
def evaluate_coco(val_dataset, model, decoder):
results, image_ids = [], []
for index in range(len(val_dataset)):
data = val_dataset[index]
scale = data['scale']
cls_heads, reg_heads, batch_anchors = model(data['img'].cuda().permute(
2, 0, 1).float().unsqueeze(dim=0))
scores, classes, boxes = decoder(cls_heads, reg_heads, batch_anchors)
scores, classes, boxes = scores.cpu(), classes.cpu(), boxes.cpu()
boxes /= scale
# make sure decode batch_size=1
# scores shape:[1,max_detection_num]
# classes shape:[1,max_detection_num]
# bboxes shape[1,max_detection_num,4]
assert scores.shape[0] == 1
scores = scores.squeeze(0)
classes = classes.squeeze(0)
boxes = boxes.squeeze(0)
# for coco_eval,we need [x_min,y_min,w,h] format pred boxes
boxes[:, 2:] -= boxes[:, :2]
for object_score, object_class, object_box in zip(
scores, classes, boxes):
object_score = float(object_score)
object_class = int(object_class)
object_box = object_box.tolist()
if object_class == -1:
break
image_result = {
'image_id':
val_dataset.image_ids[index],
'category_id':
val_dataset.find_category_id_from_coco_label(object_class),
'score':
object_score,
'bbox':
object_box,
}
results.append(image_result)
image_ids.append(val_dataset.image_ids[index])
print('{}/{}'.format(index, len(val_dataset)), end='\r')
if not len(results):
print("No target detected in test set images")
return
json.dump(results,
open('{}_bbox_results.json'.format(val_dataset.set_name), 'w'),
indent=4)
# load results in COCO evaluation tool
coco_true = val_dataset.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(
val_dataset.set_name))
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
all_eval_result = coco_eval.stats
return all_eval_result
# '''
# process annotations_GT
# change box format from XYXY to XYWH (and adding area)
for key, anno in cocoGt.anns.items():
[x1, y1, x2, y2] = anno['bbox']
x1 = min(x1, x2)
x2 = max(x1, x2)
y1 = min(y1, y2)
y2 = max(y1, y2)
width = abs(x2 - x1)
height = abs(y2 - y1)
# Don't do that
# anno['bbox'] = [x1, y1, width, height]
anno['area'] = width * height
cocoGt.anns[key] = anno
# start evaluation
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
# select N imgs to evaluate
imgIds_sort = sorted(ImgIds)
# cocoEval.params.imgIds = imgIds_sort[:500]
cocoEval.evaluate() #评价
cocoEval.accumulate() #积累
cocoEval.summarize() #总结
exit()
def evaluate_coco(model,
dataset,
coco,
eval_type="bbox",
limit=0,
image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
# Pick COCO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
# Get corresponding COCO image IDs.
coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
# Load image
image = dataset.load_image(image_id)
# Run detection
t = time.time()
r = model.detect([image], verbose=0)[0]
t_prediction += (time.time() - t)
# Convert results to COCO format
# Cast masks to uint8 because COCO tools errors out on bool
image_results = build_coco_results(dataset, coco_image_ids[i:i + 1],
r["rois"], r["class_ids"],
r["scores"],
r["masks"].astype(np.uint8))
results.extend(image_results)
# Load results. This modifies results with additional attributes.
coco_results = coco.loadRes(results)
# Evaluate
cocoEval = COCOeval(coco, coco_results, eval_type)
cocoEval.params.imgIds = coco_image_ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
# # Subclass default COCO config for video inference (batch size > 1).
# batch_size = 3
# class InferenceConfig(Config):
# NAME="VideoInf"
# GPU_COUNT = 1
# IMAGES_PER_GPU = batch_size # batch size.
# DETECTION_MIN_CONFIDENCE = 0.6
# config = InferenceConfig()
# config.display()
# #@title Process video frame-by-frame
# capture = cv2.VideoCapture("Test.mov")
# print("Processing " + "Test.mov")
# try:
# if os.path.exists("Output"):
# # Clear image cache.
# shutil.rmtree("Output")
# os.makedirs("Output")
# except OSError:
# pass
# frames = []
# frame_count = 0
# while True:
# ret, frame = capture.read()
# if not ret:
# break
# # Buffer frames into batch to fit on GPU.
# frame_count += 1
# frames.append(frame)
# if len(frames) == batch_size:
# results = model.detect(frames, verbose=0)
# # Process footage frame-by-frame.
# for i, (frame, r) in enumerate(zip(frames, results)):
# # Filter results for Formalytics features.
# frame = display_instances(frame, *filter_results(r))
# # Write processed frame back to image.
# name = '{0}.jpg'.format(frame_count + i - batch_size)
# name = os.path.join("Output", name)
# cv2.imwrite(name, frame)
# print('writing to file:{0}'.format(name))
# # Start next batch.
# frames = []
# capture.release()
# #@title Calculate video FPS
# video = cv2.VideoCapture("Test.mov");
# # Find OpenCV version
# (major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.')
# if int(major_ver) < 3 :
# fps = video.get(cv2.cv.CV_CAP_PROP_FPS)
# print("Frames per second using video.get(cv2.cv.CV_CAP_PROP_FPS): {0}".format(fps))
# else :
# fps = video.get(cv2.CAP_PROP_FPS)
# print("Frames per second using video.get(cv2.CAP_PROP_FPS) : {0}".format(fps))
# video.release();
# #@title Write video
# FPS = 60 #@param
# def make_video(outvid, images=None, fps=FPS, size=None,
# is_color=True, format="FMP4"):
# """
# Create a video from a list of images.
# @param outvid output video
# @param images list of images to use in the video
# @param fps frame per second
# @param size size of each frame
# @param is_color color
# @param format see http://www.fourcc.org/codecs.php
# @return see http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_gui/py_video_display/py_video_display.html
# The function relies on http://opencv-python-tutroals.readthedocs.org/en/latest/.
# By default, the video will have the size of the first image.
# It will resize every image to this size before adding them to the video.
# """
# from cv2 import VideoWriter, VideoWriter_fourcc, imread, resize
# fourcc = VideoWriter_fourcc(*format)
# vid = None
# for image in images:
# if not os.path.exists(image):
# raise FileNotFoundError(image)
# img = imread(image)
# if vid is None:
# if size is None:
# size = img.shape[1], img.shape[0]
# vid = VideoWriter(outvid, fourcc, float(fps), size, is_color)
# if size[0] != img.shape[1] and size[1] != img.shape[0]:
# img = resize(img, size)
# vid.write(img)
# vid.release()
# return vid
# import glob
# import os
# # Directory of images to run detection on
# images = list(glob.iglob(os.path.join("Output", '*.*')))
# # Sort the images by integer index
# images = sorted(images, key=lambda x: float(os.path.split(x)[1][:-3]))
# outvid = os.path.join("./", "out.mp4")
# make_video(outvid, images, fps=FPS)
# !ls -alh ./videos/
from fast_rcnn.nms_wrapper import nms from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval import numpy as np import skimage.io as io import pylab if __name__ == '__main__': pylab.rcParams['figure.figsize'] = (10.0, 8.0) annType = 'bbox' ground_truth = '/mnt/d/BigData/COCO/instances_train-val2014/annotations/instances_val2014.json' generated_result = '/mnt/c/Users/Lavenger/git/py-faster-rcnn/tools/result.json' cocoGt = COCO(generated_result) cocoDt = cocoGt.loadRes(generated_result) cocoEval = COCOeval(cocoGt,cocoDt) cocoEval.params.imgIds = imgIds cocoEval.params.useSegm = False cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def eval_mscoco_with_segm(cocoGT, cocoPred):
# running evaluation
cocoEval = COCOeval(cocoGT, cocoPred, "keypoints")
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def main():
# pylint: disable=import-outside-toplevel
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
parser = make_parser()
args = parser.parse_args()
if args.end_epoch == -1:
args.end_epoch = args.start_epoch
for epoch_num in range(args.start_epoch, args.end_epoch + 1):
if args.model:
model_file = args.model
else:
model_file = "log-of-{}/epoch_{}.pkl".format(
os.path.basename(args.file).split(".")[0], epoch_num)
logger.info("Load Model : %s completed", model_file)
results_list = list()
result_queue = Queue(2000)
procs = []
for i in range(args.ngpus):
proc = Process(
target=worker,
args=(
args.file,
model_file,
args.dataset_dir,
i,
args.ngpus,
result_queue,
),
)
proc.start()
procs.append(proc)
for _ in tqdm(range(5000)):
results_list.append(result_queue.get())
for p in procs:
p.join()
sys.path.insert(0, os.path.dirname(args.file))
current_network = importlib.import_module(
os.path.basename(args.file).split(".")[0])
cfg = current_network.Cfg()
all_results = DetEvaluator.format(results_list, cfg)
json_path = "log-of-{}/epoch_{}.json".format(
os.path.basename(args.file).split(".")[0], epoch_num)
all_results = json.dumps(all_results)
with open(json_path, "w") as fo:
fo.write(all_results)
logger.info("Save to %s finished, start evaluation!", json_path)
eval_gt = COCO(
os.path.join(args.dataset_dir, cfg.test_dataset["name"],
cfg.test_dataset["ann_file"]))
eval_dt = eval_gt.loadRes(json_path)
cocoEval = COCOeval(eval_gt, eval_dt, iouType="bbox")
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
metrics = [
"AP",
"[email protected]",
"[email protected]",
"APs",
"APm",
"APl",
"AR@1",
"AR@10",
"AR@100",
"ARs",
"ARm",
"ARl",
]
logger.info("mmAP".center(32, "-"))
for i, m in enumerate(metrics):
logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i])
logger.info("-" * 32)
def main():
args = get_args()
cocoGt = COCO(
os.path.join(args.coco_dir, "annotations/instances_val2017.json"))
if args.use_inv_map:
inv_map = [0
] + cocoGt.getCatIds() # First label in inv_map is not used
with open(args.mlperf_accuracy_file, "r") as f:
results = json.load(f)
detections = []
image_ids = set()
seen = set()
no_results = 0
if args.remove_48_empty_images:
im_ids = []
for i in cocoGt.getCatIds():
im_ids += cocoGt.catToImgs[i]
im_ids = list(set(im_ids))
image_map = [cocoGt.imgs[id] for id in im_ids]
else:
image_map = cocoGt.dataset["images"]
for j in results:
idx = j['qsl_idx']
# de-dupe in case loadgen sends the same image multiple times
if idx in seen:
continue
seen.add(idx)
# reconstruct from mlperf accuracy log
# what is written by the benchmark is an array of float32's:
# id, box[0], box[1], box[2], box[3], score, detection_class
# note that id is a index into instances_val2017.json, not the actual image_id
data = np.frombuffer(bytes.fromhex(j['data']), np.float32)
if len(data) < 7:
# handle images that had no results
image = image_map[idx]
# by adding the id to image_ids we make pycoco aware of the no-result image
image_ids.add(image["id"])
no_results += 1
if args.verbose:
print("no results: {}, idx={}".format(image["coco_url"], idx))
continue
for i in range(0, len(data), 7):
image_idx, ymin, xmin, ymax, xmax, score, label = data[i:i + 7]
image = image_map[idx]
image_idx = int(image_idx)
if image_idx != idx:
print(
"ERROR: loadgen({}) and payload({}) disagree on image_idx".
format(idx, image_idx))
image_id = image["id"]
height, width = image["height"], image["width"]
ymin *= height
xmin *= width
ymax *= height
xmax *= width
loc = os.path.join(args.coco_dir, "val2017", image["file_name"])
label = int(label)
if args.use_inv_map:
label = inv_map[label]
# pycoco wants {imageID,x1,y1,w,h,score,class}
detections.append({
"image_id":
image_id,
"image_loc":
loc,
"category_id":
label,
"bbox": [
float(xmin),
float(ymin),
float(xmax - xmin),
float(ymax - ymin)
],
"score":
float(score)
})
image_ids.add(image_id)
with open(args.output_file, "w") as fp:
json.dump(detections, fp, sort_keys=True, indent=4)
cocoDt = cocoGt.loadRes(
args.output_file) # Load from file to bypass error with Python3
cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox')
cocoEval.params.imgIds = list(image_ids)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("mAP={:.3f}%".format(100. * cocoEval.stats[0]))
if args.verbose:
print("found {} results".format(len(results)))
print("found {} images".format(len(image_ids)))
print("found {} images with no results".format(no_results))
print("ignored {} dupes".format(len(results) - len(seen)))
def evaluate_mAP(
res_file,
ann_type='bbox',
ann_file='./data/coco/annotations/person_keypoints_val2017.json',
silence=True):
"""Evaluate mAP result for coco dataset.
Parameters
----------
res_file: str
Path to result json file.
ann_type: str
annotation type, including: `bbox`, `segm`, `keypoints`.
ann_file: str
Path to groundtruth file.
silence: bool
True: disable running log.
"""
class NullWriter(object):
def write(self, arg):
pass
# ann_file = os.path.join('./data/coco/annotations/', ann_file)
if silence:
nullwrite = NullWriter()
oldstdout = sys.stdout
sys.stdout = nullwrite # disable output
cocoGt = COCO(ann_file)
cocoDt = cocoGt.loadRes(res_file)
cocoEval = COCOeval(cocoGt, cocoDt, ann_type)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if silence:
sys.stdout = oldstdout # enable output
if isinstance(cocoEval.stats[0], dict):
stats_names = [
'AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
parts = ['body', 'face', 'hand', 'fullbody']
info = {}
for i, part in enumerate(parts):
info[part] = cocoEval.stats[i][part][0]
return info
else:
stats_names = [
'AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
info_str = {}
for ind, name in enumerate(stats_names):
info_str[name] = cocoEval.stats[ind]
return info_str['AP']
def benchmark_model(frozen_graph,
images_dir,
annotation_path,
batch_size=1,
image_shape=None,
num_images=4096,
tmp_dir='.benchmark_model_tmp_dir',
remove_tmp_dir=True,
output_path=None,
display_every=100):
"""Computes accuracy and performance statistics
Computes accuracy and performance statistics by executing over many images
from the MSCOCO dataset defined by images_dir and annotation_path.
Args
----
frozen_graph: A GraphDef representing the object detection model to
test. Alternatively, a string representing the path to the saved
frozen graph.
images_dir: A string representing the path of the COCO images
directory.
annotation_path: A string representing the path of the COCO annotation
file.
batch_size: An integer representing the batch size to use when feeding
images to the model.
image_shape: An optional tuple of integers representing a fixed shape
to resize all images before testing.
num_images: An integer representing the number of images in the
dataset to evaluate with.
tmp_dir: A string representing the path where the function may create
a temporary directory to store intermediate files.
output_path: An optional string representing a path to store the
statistics in JSON format.
display_every: int, print log every display_every iteration
Returns
-------
statistics: A named dictionary of accuracy and performance statistics
computed for the model.
"""
if os.path.exists(tmp_dir):
if not remove_tmp_dir:
raise RuntimeError('Temporary directory exists; %s' % tmp_dir)
subprocess.call(['rm', '-rf', tmp_dir])
if batch_size > 1 and image_shape is None:
raise RuntimeError(
'Fixed image shape must be provided for batch size > 1')
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
coco = COCO(annotation_file=annotation_path)
# get list of image ids to use for evaluation
image_ids = coco.getImgIds()
if num_images > len(image_ids):
print(
'Num images provided %d exceeds number in dataset %d, using %d images instead'
% (num_images, len(image_ids), len(image_ids)))
num_images = len(image_ids)
image_ids = image_ids[0:num_images]
# load frozen graph from file if string, otherwise must be GraphDef
if isinstance(frozen_graph, str):
frozen_graph_path = frozen_graph
frozen_graph = tf.GraphDef()
with open(frozen_graph_path, 'rb') as f:
frozen_graph.ParseFromString(f.read())
elif not isinstance(frozen_graph, tf.GraphDef):
raise TypeError('Expected frozen_graph to be GraphDef or str')
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
coco_detections = [] # list of all bounding box detections in coco format
runtimes = [] # list of runtimes for each batch
image_counts = [] # list of number of images in each batch
with tf.Graph().as_default() as tf_graph:
with tf.Session(config=tf_config) as tf_sess:
tf.import_graph_def(frozen_graph, name='')
tf_input = tf_graph.get_tensor_by_name(INPUT_NAME + ':0')
tf_boxes = tf_graph.get_tensor_by_name(BOXES_NAME + ':0')
tf_classes = tf_graph.get_tensor_by_name(CLASSES_NAME + ':0')
tf_scores = tf_graph.get_tensor_by_name(SCORES_NAME + ':0')
tf_num_detections = tf_graph.get_tensor_by_name(
NUM_DETECTIONS_NAME + ':0')
# load batches from coco dataset
for image_idx in range(0, len(image_ids), batch_size):
batch_image_ids = image_ids[image_idx:image_idx + batch_size]
batch_images = []
batch_coco_images = []
# read images from file
for image_id in batch_image_ids:
coco_img = coco.imgs[image_id]
batch_coco_images.append(coco_img)
image_path = os.path.join(images_dir,
coco_img['file_name'])
image = _read_image(image_path, image_shape)
batch_images.append(image)
# run once outside of timing to initialize
if image_idx == 0:
boxes, classes, scores, num_detections = tf_sess.run(
[tf_boxes, tf_classes, tf_scores, tf_num_detections],
feed_dict={tf_input: batch_images})
# execute model and compute time difference
t0 = time.time()
boxes, classes, scores, num_detections = tf_sess.run(
[tf_boxes, tf_classes, tf_scores, tf_num_detections],
feed_dict={tf_input: batch_images})
t1 = time.time()
# log runtime and image count
runtimes.append(float(t1 - t0))
if len(runtimes) % display_every == 0:
print(" step %d/%d, iter_time(ms)=%.4f" %
(len(runtimes),
(len(image_ids) + batch_size - 1) / batch_size,
np.mean(runtimes) * 1000))
image_counts.append(len(batch_images))
# add coco detections for this batch to running list
batch_coco_detections = []
for i, image_id in enumerate(batch_image_ids):
image_width = batch_coco_images[i]['width']
image_height = batch_coco_images[i]['height']
for j in range(int(num_detections[i])):
bbox = boxes[i][j]
bbox_coco_fmt = [
bbox[1] * image_width, # x0
bbox[0] * image_height, # x1
(bbox[3] - bbox[1]) * image_width, # width
(bbox[2] - bbox[0]) * image_height, # height
]
coco_detection = {
'image_id': image_id,
'category_id': int(classes[i][j]),
'bbox': bbox_coco_fmt,
'score': float(scores[i][j])
}
coco_detections.append(coco_detection)
# write coco detections to file
subprocess.call(['mkdir', '-p', tmp_dir])
coco_detections_path = os.path.join(tmp_dir, 'coco_detections.json')
with open(coco_detections_path, 'w') as f:
json.dump(coco_detections, f)
# compute coco metrics
cocoDt = coco.loadRes(coco_detections_path)
eval = COCOeval(coco, cocoDt, 'bbox')
eval.params.imgIds = image_ids
eval.evaluate()
eval.accumulate()
eval.summarize()
statistics = {
'map': eval.stats[0],
'avg_latency_ms': 1000.0 * np.mean(runtimes),
'avg_throughput_fps': np.sum(image_counts) / np.sum(runtimes),
'runtimes_ms': [1000.0 * r for r in runtimes]
}
if output_path is not None:
subprocess.call(['mkdir', '-p', os.path.dirname(output_path)])
with open(output_path, 'w') as f:
json.dump(statistics, f)
subprocess.call(['rm', '-rf', tmp_dir])
return statistics
def main():
# pylint: disable=import-outside-toplevel,too-many-branches,too-many-statements
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
parser = make_parser()
args = parser.parse_args()
current_network = import_from_file(args.file)
cfg = current_network.Cfg()
if args.weight_file:
args.start_epoch = args.end_epoch = -1
else:
if args.start_epoch == -1:
args.start_epoch = cfg.max_epoch - 1
if args.end_epoch == -1:
args.end_epoch = args.start_epoch
assert 0 <= args.start_epoch <= args.end_epoch < cfg.max_epoch
for epoch_num in range(args.start_epoch, args.end_epoch + 1):
if args.weight_file:
weight_file = args.weight_file
else:
weight_file = "log-of-{}/epoch_{}.pkl".format(
os.path.basename(args.file).split(".")[0], epoch_num)
if args.ngpus > 1:
master_ip = "localhost"
port = dist.get_free_ports(1)[0]
dist.Server(port)
result_list = []
result_queue = Queue(2000)
procs = []
for i in range(args.ngpus):
proc = Process(
target=worker,
args=(
current_network,
weight_file,
args.dataset_dir,
master_ip,
port,
args.ngpus,
i,
result_queue,
),
)
proc.start()
procs.append(proc)
num_imgs = dict(coco=5000, cocomini=5000, objects365=30000)
for _ in tqdm(range(num_imgs[cfg.test_dataset["name"]])):
result_list.append(result_queue.get())
for p in procs:
p.join()
else:
result_list = []
worker(current_network, weight_file, args.dataset_dir, None, None,
1, 0, result_list)
total_time = sum([x["perf_time"] for x in result_list])
average_time = total_time / len(result_list)
fps = 1.0 / average_time
logger.info("average inference speed: {:.4}s / iter, fps:{:.3}".format(
average_time, fps))
all_results = DetEvaluator.format(result_list, cfg)
json_path = "log-of-{}/epoch_{}.json".format(
os.path.basename(args.file).split(".")[0], epoch_num)
all_results = json.dumps(all_results)
with open(json_path, "w") as fo:
fo.write(all_results)
logger.info("Save to %s finished, start evaluation!", json_path)
eval_gt = COCO(
os.path.join(args.dataset_dir, cfg.test_dataset["name"],
cfg.test_dataset["ann_file"]))
eval_dt = eval_gt.loadRes(json_path)
cocoEval = COCOeval(eval_gt, eval_dt, iouType="bbox")
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
metrics = [
"AP",
"[email protected]",
"[email protected]",
"APs",
"APm",
"APl",
"AR@1",
"AR@10",
"AR@100",
"ARs",
"ARm",
"ARl",
]
logger.info("mmAP".center(32, "-"))
for i, m in enumerate(metrics):
logger.info("|\t%s\t|\t%.03f\t|", m, cocoEval.stats[i])
logger.info("-" * 32)
def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
# Pick COCO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
# Get corresponding COCO image IDs.
coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
# Load image
image = dataset.load_image(image_id)
# Run detection
t = time.time()
r = model.detect([image])[0]
t_prediction += (time.time() - t)
# Convert results to COCO format
image_results = build_coco_results(dataset, coco_image_ids[i:i + 1],
r["rois"], r["class_ids"],
r["scores"], r["masks"])
results.extend(image_results)
# Load results. This modifies results with additional attributes.
coco_results = coco.loadRes(results)
# Evaluate
cocoEval = COCOeval(coco, coco_results, eval_type)
cocoEval.params.imgIds = coco_image_ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
def evaluate_coco(dataset, model, threshold=0.05):
model.eval()
device = [p.device for p in model.parameters()][0]
with torch.no_grad():
# start collecting results
results = []
image_ids = []
for index in range(len(dataset)):
data = dataset[index]
scale = data['scale']
# run network
#TODO: question, why is this permuted bellow?
scores, labels, boxes = model(
data.image.permute(
2, 0, 1).to(device=device).float().unsqueeze(dim=0))
scores = scores.cpu()
labels = labels.cpu()
boxes = boxes.cpu()
# correct boxes for image scale
boxes /= scale
if boxes.shape[0] > 0:
# change to (x, y, w, h) (MS COCO standard)
boxes[:, 2] -= boxes[:, 0]
boxes[:, 3] -= boxes[:, 1]
# compute predicted labels and scores
#for box, score, label in zip(boxes[0], scores[0], labels[0]):
for box_id in range(boxes.shape[0]):
score = float(scores[box_id])
label = int(labels[box_id])
box = boxes[box_id, :]
# scores are sorted, so we can break
if score < threshold:
break
# append detection for each positively labeled class
image_result = {
'image_id': dataset.image_ids[index],
'category_id': dataset.label_to_coco_label(label),
'score': float(score),
'bbox': box.tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(dataset.image_ids[index])
# print progress
print('{}/{}'.format(index, len(dataset)), end='\r')
if not len(results):
return
# write output
json.dump(results,
open('{}_bbox_results.json'.format(dataset.set_name), 'w'),
indent=4)
# load results in COCO evaluation tool
coco_true = dataset.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(
dataset.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
model.train()
return