def evaluate(model, data_loader, cfg, device, logger=None, **kwargs):
""" finished, tested
"""
# cpu_device = torch.device("cpu")
model.eval()
# header = 'Test:'
coco = convert_to_coco_api(data_loader.dataset, bbox_fmt='coco')
coco_evaluator = CocoEvaluator(coco, iou_types = ["bbox"], bbox_fmt='coco')
for images, targets in data_loader:
model_input = [[cv2.resize(img, (cfg.w, cfg.h))] for img in images]
model_input = np.concatenate(model_input, axis=0)
model_input = model_input.transpose(0, 3, 1, 2)
model_input = torch.from_numpy(model_input).div(255.0)
model_input = model_input.to(device)
targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
if torch.cuda.is_available():
torch.cuda.synchronize()
model_time = time.time()
outputs = model(model_input)
# outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
outputs = outputs.cpu().detach().numpy()
res = {}
for img, target, output in zip(images, targets, outputs):
img_height, img_width = img.shape[:2]
boxes = output[...,:4].copy() # output boxes in yolo format
boxes[...,:2] = boxes[...,:2] - boxes[...,2:]/2 # to coco format
boxes[...,0] = boxes[...,0]*img_width
boxes[...,1] = boxes[...,1]*img_height
boxes[...,2] = boxes[...,2]*img_width
boxes[...,3] = boxes[...,3]*img_height
boxes = torch.as_tensor(boxes, dtype=torch.float32)
confs = output[...,4:].copy()
labels = np.argmax(confs, axis=1).flatten()
labels = torch.as_tensor(labels, dtype=torch.int64)
scores = np.max(confs, axis=1).flatten()
scores = torch.as_tensor(scores, dtype=torch.float32)
res[target["image_id"].item()] = {
"boxes": boxes,
"scores": scores,
"labels": labels,
}
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_time = time.time() - evaluator_time
# gather the stats from all processes
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
return coco_evaluator
def evaluate_nms(model, data_loader, cfg, device, human_patch=False, json_gt=None, **kwargs):
""" finished, tested
"""
# cpu_device = torch.device("cpu")
model.eval()
# header = 'Test:'
if json_gt is None:
coco = convert_to_coco_api(data_loader.dataset, bbox_fmt='coco')
coco_evaluator = CocoEvaluator(coco, iou_types = ["bbox"], bbox_fmt='coco')
else:
coco_gt = COCO(json_gt)
result_json = []
for images, targets in tqdm.tqdm(data_loader, desc='testing'):
model_input = [[cv2.resize(img, (cfg.w, cfg.h))] for img in images]
model_input = np.concatenate(model_input, axis=0)
model_input = model_input.transpose(0, 3, 1, 2)
model_input = torch.from_numpy(model_input).div(255.0)
model_input = model_input.to(device)
#targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
if torch.cuda.is_available():
torch.cuda.synchronize()
model_time = time.time()
outputs = model(model_input)
outputs = utils.post_processing(conf_thresh=0.001, nms_thresh=0.5, output=outputs)
# outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
# outputs = outputs.cpu().detach().numpy()
res = {}
# for img, target, output in zip(images, targets, outputs):
for img, target, output in zip(images, targets, outputs):
img_height, img_width = img.shape[:2]
#human_box = target['human_box']
# boxes = output[...,:4].copy() # output boxes in yolo format
if len(output) == 0:
continue
boxes = output[:,:4]
scores = output[:,-2]
labels = output[:,-1]
boxes[...,2:] = boxes[...,2:] - boxes[...,:2] # Transform [x1, y1, x2, y2] to [x1, y1, w, h]
if human_patch:
human_box = target['human_box']#.cpu().detach().numpy()
boxes[...,0] = boxes[...,0]*img_width + human_box[0]
boxes[...,1] = boxes[...,1]*img_height + human_box[1]
else:
boxes[...,0] = boxes[...,0]*img_width
boxes[...,1] = boxes[...,1]*img_height
boxes[...,2] = boxes[...,2]*img_width
boxes[...,3] = boxes[...,3]*img_height
if json_gt is None:
boxes = torch.as_tensor(boxes, dtype=torch.float32)
# confs = output[...,4:].copy()
labels = torch.as_tensor(labels, dtype=torch.int64)
scores = torch.as_tensor(scores, dtype=torch.float32)
res[target["image_id"]] = {
"boxes": boxes.unsqueeze(1),
"scores": scores,
"labels": labels,
}
else:
for box, label, score in zip(boxes, labels, scores):
single_result = {'image_id': int(target['image_id']),
'bbox': box.tolist(),
'category_id': int(label+1),
'score':float(score)}
result_json.append(single_result)
evaluator_time = time.time()
if json_gt is None:
if len(res) != 0:
coco_evaluator.update(res)
evaluator_time = time.time() - evaluator_time
if json_gt is None:
# gather the stats from all processes
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
return coco_evaluator
else:
with open('temp_result.json', 'w') as f:
json.dump(result_json, f)
coco_dt = coco_gt.loadRes('temp_result.json')
cocoEval = COCOeval(coco_gt,coco_dt,'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
return cocoEval
def evaluate(model, data_loader, cfg, device, logger=None, **kwargs):
""" finished, tested
"""
# cpu_device = torch.device("cpu")
model.eval()
# header = 'Test:'
coco = convert_to_coco_api(data_loader.dataset, bbox_fmt='coco')
coco_evaluator = CocoEvaluator(coco, iou_types=["bbox"], bbox_fmt='coco')
for images, targets in data_loader:
model_input = [[cv2.resize(img, (cfg.w, cfg.h))] for img in images]
model_input = np.concatenate(model_input, axis=0)
model_input = model_input.transpose(0, 3, 1, 2)
model_input = torch.from_numpy(model_input).div(255.0)
model_input = model_input.to(device)
targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
if torch.cuda.is_available():
torch.cuda.synchronize()
model_time = time.time()
outputs = model(model_input)
# outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
# outputs = outputs.cpu().detach().numpy()
res = {}
# for img, target, output in zip(images, targets, outputs):
for img, target, boxes, confs in zip(images, targets, outputs[0],
outputs[1]):
img_height, img_width = img.shape[:2]
# boxes = output[...,:4].copy() # output boxes in yolo format
boxes = boxes.squeeze(2).cpu().detach().numpy()
boxes[..., 2:] = boxes[..., 2:] - boxes[
..., :2] # Transform [x1, y1, x2, y2] to [x1, y1, w, h]
boxes[..., 0] = boxes[..., 0] * img_width
boxes[..., 1] = boxes[..., 1] * img_height
boxes[..., 2] = boxes[..., 2] * img_width
boxes[..., 3] = boxes[..., 3] * img_height
boxes = torch.as_tensor(boxes, dtype=torch.float32)
# confs = output[...,4:].copy()
confs = confs.cpu().detach().numpy()
labels = np.argmax(confs, axis=1).flatten()
labels = torch.as_tensor(labels, dtype=torch.int64)
scores = np.max(confs, axis=1).flatten()
scores = torch.as_tensor(scores, dtype=torch.float32)
res[target["image_id"].item()] = {
"boxes": boxes,
"scores": scores,
"labels": labels,
}
debug = kwargs.get("debug", [])
if isinstance(debug, str):
debug = [debug]
debug = [item.lower() for item in debug]
if 'iou' in debug:
from tool.utils_iou_test import bboxes_iou_test
ouput_boxes = np.array(
post_processing(None, 0.5, 0.5, outputs)[0])[..., :4]
img_height, img_width = images[0].shape[:2]
ouput_boxes[..., 0] = ouput_boxes[..., 0] * img_width
ouput_boxes[..., 1] = ouput_boxes[..., 1] * img_height
ouput_boxes[..., 2] = ouput_boxes[..., 2] * img_width
ouput_boxes[..., 3] = ouput_boxes[..., 3] * img_height
# coco format to yolo format
truth_boxes = targets[0]['boxes'].numpy().copy()
truth_boxes[
..., :2] = truth_boxes[..., :2] + truth_boxes[..., 2:] / 2
iou = bboxes_iou_test(torch.Tensor(ouput_boxes),
torch.Tensor(truth_boxes),
fmt='yolo')
print(f"iou of first image = {iou}")
if len(debug) > 0:
return
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_time = time.time() - evaluator_time
# gather the stats from all processes
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
return coco_evaluator
def evaluate_nms_patch(model, data_loader, cfg, device, **kwargs):
""" finished, tested
"""
# cpu_device = torch.device("cpu")
model.eval()
# header = 'Test:'
coco = convert_to_coco_api(data_loader.dataset, bbox_fmt='coco')
coco_evaluator = CocoEvaluator(coco, iou_types = ["bbox"], bbox_fmt='coco')
for images, targets in tqdm.tqdm(data_loader):
model_input = [[cv2.resize(img, (cfg.w, cfg.h))] for img in images]
model_input = np.concatenate(model_input, axis=0)
model_input = model_input.transpose(0, 3, 1, 2)
model_input = torch.from_numpy(model_input).div(255.0)
model_input = model_input.to(device)
targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
if torch.cuda.is_available():
torch.cuda.synchronize()
model_time = time.time()
outputs = model(model_input)
outputs = utils.post_processing(conf_thresh=0.001, nms_thresh=0.5, output=outputs)
# outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
# outputs = outputs.cpu().detach().numpy()
res = {}
# for img, target, output in zip(images, targets, outputs):
for img, target, output in zip(images, targets, outputs):
img_height, img_width = img.shape[:2]
#human_box = target['human_box']
# boxes = output[...,:4].copy() # output boxes in yolo format
boxes = output[:,:4]
scores = output[:,-2]
labels = output[:,-1]
human_box = target['human_box'].cpu().detach().numpy()
boxes[...,2:] = boxes[...,2:] - boxes[...,:2] # Transform [x1, y1, x2, y2] to [x1, y1, w, h]
boxes[...,0] = boxes[...,0]*img_width + human_box[0]
boxes[...,1] = boxes[...,1]*img_height + human_box[1]
boxes[...,2] = boxes[...,2]*img_width
boxes[...,3] = boxes[...,3]*img_height
boxes = torch.as_tensor(boxes, dtype=torch.float32).unsqueeze(0)
# confs = output[...,4:].copy()
labels = torch.as_tensor(labels, dtype=torch.int64)
scores = torch.as_tensor(scores, dtype=torch.float32)
res[target["image_id"].item()] = {
"boxes": boxes,
"scores": scores,
"labels": labels,
}
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_time = time.time() - evaluator_time
# gather the stats from all processes
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
return coco_evaluator
def evaluate_all(device=None):
"""
"""
device = device or torch.device("cuda")
print(f"device = {device}")
val_dataset = ACNE04(label_path=Cfg.val_label, cfg=Cfg, train=False)
val_loader = DataLoader(
dataset=val_dataset,
batch_size=Cfg.batch // Cfg.subdivisions,
shuffle=True,
num_workers=8,
pin_memory=True,
drop_last=True,
collate_fn=val_collate,
)
coco = convert_to_coco_api(val_loader.dataset, bbox_fmt='coco')
all_models = glob.glob(os.path.join(Cfg.checkpoints, "*.pth"))
all_models.sort(key=lambda fp: int(
os.path.splitext(os.path.basename(fp))[0].replace("Yolov4_epoch", "")))
state_dict = torch.load(model_path, map_location=device)
for model_path in all_models:
print(f"eval on {os.path.splitext(os.path.basename(model_path))[0]}")
model = Yolov4(None, 1, True)
model.load_state_dict(state_dict)
model.to(device)
model.eval()
coco_evaluator = CocoEvaluator(coco, iou_types=["bbox"])
for images, targets in val_loader:
model_input = [[cv2.resize(img, (Cfg.w, Cfg.h))] for img in images]
model_input = np.concatenate(model_input, axis=0)
model_input = model_input.transpose(0, 3, 1, 2)
model_input = torch.from_numpy(model_input).div(255.0)
model_input = model_input.to(device)
targets = [{k: v.to(device)
for k, v in t.items()} for t in targets]
if torch.cuda.is_available():
torch.cuda.synchronize()
model_time = time.time()
outputs = model(model_input)
# outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
outputs = outputs.cpu().detach().numpy()
res = {}
for img, target, output in zip(images, targets, outputs):
img_height, img_width = img.shape[:2]
boxes = output[..., :4].copy() # output boxes in yolo format
boxes[..., :2] = boxes[
..., :2] - boxes[..., 2:] / 2 # to coco format
boxes[..., 0] = boxes[..., 0] * img_width
boxes[..., 1] = boxes[..., 1] * img_height
boxes[..., 2] = boxes[..., 2] * img_width
boxes[..., 3] = boxes[..., 3] * img_height
boxes = torch.as_tensor(boxes, dtype=torch.float32)
labels = torch.as_tensor(np.zeros((len(output), )),
dtype=torch.int64)
scores = torch.as_tensor(output[..., -1], dtype=torch.float32)
res[target["image_id"].item()] = {
"boxes": boxes,
"scores": scores,
"labels": labels,
}
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_time = time.time() - evaluator_time
# gather the stats from all processes
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
del model
del coco_evaluator