def evaluate(predictor, test_ds):
"""Evaluation of the validation set
Keyword arguments:
- predictor: model after training with the respective weights
- data_loader: validaton set in the loader format
- device: device on which the network will be evaluated
"""
coco = convert_to_coco_api(test_ds)
coco_evaluator = CocoEvaluator(coco)
evaluator_times = []
proc_times = []
for i in range(len(test_ds)):
image, targets = test_ds[i]
init = time.time()
boxes, labels, probs = predictor.predict(image, 10, 0.2)
proc_times.append(time.time() - init)
if boxes.size()[0] == 0:
continue
outputs = {'boxes': boxes, 'labels': labels, 'scores': probs}
res = {targets['image_id'].item(): outputs}
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_times.append(time.time() - evaluator_time)
print("Averaged stats:", np.mean(evaluator_times))
print("Averaged proc time:", np.mean(proc_times))
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
return coco_evaluator
def evaluate(model,data_loader,device):
"""Evaluation of the validation set
Keyword arguments:
- model: model after training with the respective weights
- data_loader: validaton set in the loader format
- device: device on which the network will be evaluated
"""
cpu_device = torch.device("cpu")
model.eval()
coco = convert_to_coco_api(data_loader.dataset)
coco_evaluator = CocoEvaluator(coco)
evaluator_times = []
proc_times = []
for image, targets in data_loader:
image, targets = transform_inputs(image, targets, device)
init = time.time()
outputs = model(image)
proc_times.append(time.time() - init)
outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]
res = {target["image_id"].item(): output for target, output in zip(targets, outputs)}
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_times.append(time.time() - evaluator_time)
print("Averaged stats:", np.mean(evaluator_times))
print("Averaged proc time:", np.mean(proc_times))
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
return coco_evaluator
def evaluate(model, data_loader, device):
"""Evaluation of the validation set
Keyword arguments:
- model: model after training with the respective weights
- data_loader: validaton set in the loader format
- device: device on which the network will be evaluated
"""
cpu_device = torch.device("cpu")
model.eval().to(device)
coco = convert_to_coco_api(data_loader.dataset)
coco_evaluator = CocoEvaluator(coco)
evaluator_times = []
proc_times = []
for images, targets in data_loader:
res = {}
images, targets = transform_inputs(images, targets, device)
images_eval = [image.float() / 255 for image in images] #normalized
batch = torch.stack(images_eval)
init = time.time()
inf_out, eval_out = model(batch)
proc_times.append(time.time() - init)
output = non_max_suppression(inf_out, conf_thres=0.001, iou_thres=0.6)
for si, pred in enumerate(output):
height, width = images[si].shape[1:]
if pred is None:
box = torch.tensor([[0, 0, 0, 0]])
res.update({
targets[si]["image_id"].item(): {
"boxes": box,
"labels": torch.tensor([1]),
"scores": torch.tensor([0])
}
})
else:
clip_coords(pred, (height, width))
box = pred[:, :4].clone()
res.update({
targets[si]["image_id"].item(): {
"boxes": box,
"labels": pred[:, 5],
"scores": pred[:, 4]
}
})
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_times.append(time.time() - evaluator_time)
print("Averaged stats:", np.mean(evaluator_times))
print("Averaged proc time:", np.mean(proc_times))
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
return coco_evaluator
def on_evaluation_started(engine):
model.eval()
engine.state.coco_evaluator = CocoEvaluator(coco_api_val_dataset)