def evaluate(test_loader, model):
""" Evaluate.
Args:
test_loader: DataLoader for test data
model: model
"""
model.eval()
det_boxes = list()
det_labels = list()
det_scores = list()
true_boxes = list()
true_labels = list()
true_difficulties = list()
for i, (images, boxes, labels, difficulties) in enumerate(test_loader):
print(f'Iters: [{i}/{length}]')
images = jt.array(images)
predicted_locs, predicted_scores = model(images)
det_boxes_batch, det_labels_batch, det_scores_batch = model.detect_objects(predicted_locs, predicted_scores, min_score=0.01, max_overlap=0.45, top_k=200)
det_boxes.extend(det_boxes_batch)
det_labels.extend(det_labels_batch)
det_scores.extend(det_scores_batch)
true_boxes.extend(boxes)
true_labels.extend(labels)
true_difficulties.extend(difficulties)
APs, mAP = calculate_mAP(det_boxes, det_labels, det_scores, true_boxes, true_labels, true_difficulties)
pp.pprint(APs)
print('\nMean Average Precision (mAP): %.3f' % mAP)
def evaluate():
checkpoint_path = os.path.join(args.model_root, args.model_name)
checkpoint = torch.load(checkpoint_path, map_location=device)
model = SSD300(n_classes=len(label_map), device=device).to(device)
model.load_state_dict(checkpoint['model'])
transform = Transform(size=(300, 300), train=False)
test_dataset = VOCDataset(root=args.data_root,
image_set=args.image_set,
transform=transform,
keep_difficult=True)
test_loader = DataLoader(dataset=test_dataset,
collate_fn=collate_fn,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False,
pin_memory=True)
detected_bboxes = []
detected_labels = []
detected_scores = []
true_bboxes = []
true_labels = []
true_difficulties = []
model.eval()
with torch.no_grad():
bar = tqdm(test_loader, desc='Evaluate the model')
for i, (images, bboxes, labels, difficulties) in enumerate(bar):
images = images.to(device)
bboxes = [b.to(device) for b in bboxes]
labels = [l.to(device) for l in labels]
difficulties = [d.to(device) for d in difficulties]
predicted_bboxes, predicted_scores = model(images)
_bboxes, _labels, _scores = model.detect_objects(predicted_bboxes,
predicted_scores,
min_score=0.01,
max_overlap=0.45,
top_k=200)
detected_bboxes += _bboxes
detected_labels += _labels
detected_scores += _scores
true_bboxes += bboxes
true_labels += labels
true_difficulties += difficulties
all_ap, mean_ap = calculate_mAP(detected_bboxes,
detected_labels,
detected_scores,
true_bboxes,
true_labels,
true_difficulties,
device=device)
pretty_printer = PrettyPrinter()
pretty_printer.pprint(all_ap)
print('Mean Average Precision (mAP): %.4f' % mean_ap)
def evaluate(test_loader, model, pp, device):
"""
Evaluate.
:param test_loader: DataLoader for test data
:param model: model
"""
# Make sure it's in eval mode
model.eval()
# Lists to store detected and true boxes, labels, scores
det_boxes = list()
det_labels = list()
det_scores = list()
true_boxes = list()
true_labels = list()
true_difficulties = list(
) # it is necessary to know which objects are 'difficult', see 'calculate_mAP' in utils.py
with torch.no_grad():
# Batches
for i, (images, boxes, labels, difficulties) in enumerate(test_loader):
images = images.to(device) # (N, 3, 300, 300)
# Forward prop.
predicted_locs, predicted_scores = model(images)
# Detect objects in SSD output
det_boxes_batch, det_labels_batch, det_scores_batch = model.detect_objects(
predicted_locs,
predicted_scores,
min_score=0.01,
max_overlap=0.45,
top_k=200,
device=device)
# Evaluation MUST be at min_score=0.01, max_overlap=0.45, top_k=200 for fair comparision with the paper's results and other repos
# Store this batch's results for mAP calculation
boxes = [b.to(device) for b in boxes]
labels = [l.to(device) for l in labels]
difficulties = [d.to(device) for d in difficulties]
det_boxes.extend(det_boxes_batch)
det_labels.extend(det_labels_batch)
det_scores.extend(det_scores_batch)
true_boxes.extend(boxes)
true_labels.extend(labels)
true_difficulties.extend(difficulties)
# Calculate mAP
APs, mAP = calculate_mAP(det_boxes, det_labels, det_scores, true_boxes,
true_labels, true_difficulties, device)
# Print AP for each class
pp.pprint(APs)
print('\nMean Average Precision (mAP): %.3f' % mAP)
def evaluate(test_loader, model, label_map, rev_label_map, device):
model.eval()
det_boxes = list()
det_labels = list()
det_scores = list()
true_boxes = list()
true_labels = list()
true_difficulties = list()
with torch.no_grad():
# Batches
for i, (images, boxes, labels,
difficulties) in enumerate(tqdm(test_loader,
desc='Evaluating')):
images = images.to(device) # (N, 3, 300, 300)
# Forward prop.
predicted_locs, predicted_scores = model(images)
# Detect objects in SSD output
det_boxes_batch, det_labels_batch, det_scores_batch = model.detect_objects(
predicted_locs,
predicted_scores,
min_score=0.01,
max_overlap=0.45,
top_k=200)
boxes = [b.to(device) for b in boxes]
labels = [l.to(device) for l in labels]
difficulties = [d.to(device) for d in difficulties]
det_boxes.extend(det_boxes_batch)
det_labels.extend(det_labels_batch)
det_scores.extend(det_scores_batch)
true_boxes.extend(boxes)
true_labels.extend(labels)
true_difficulties.extend(difficulties)
APs, mAP = calculate_mAP(label_map, rev_label_map, device, det_boxes,
det_labels, det_scores, true_boxes,
true_labels, true_difficulties)
pp = PrettyPrinter()
pp.pprint(APs)
print('\nMean Average Precision (mAP): %.3f' % mAP)
def test(self):
self.model.to(self.device)
self.model.eval()
list_test_batch_loss = []
for batch_idx, (images, labels) in enumerate(self.test_dataloader):
images, labels = images.to(self.device), labels.to(self.device)
output = self.model(images)
with torch.no_grad():
if self.args.agg_method == 'fedavg':
if self.loss_function == 'hashing loss':
loss = hashing_loss(output, labels,
2 * self.args.num_binary,
self.args.alpha)
list_test_batch_loss.append(loss)
mAP = calculate_mAP(self.model, self.train_dataloader,
self.test_dataloader)
print('Test loss: ',
sum(list_test_batch_loss) / len(list_test_batch_loss))
return mAP
def evaluate(test_loader, model, optimizer, config):
"""
Evaluate.
:param test_loader: DataLoader for test data
:param model: model
"""
# Make sure it's in eval mode
# torch.cuda.empty_cache()
model.eval()
pp = pprint.PrettyPrinter()
# Lists to store detected and true boxes, labels, scores
det_boxes = list()
det_labels = list()
det_scores = list()
true_boxes = list()
true_labels = list()
true_difficulties = list()
detect_speed = list()
with torch.no_grad():
# Batches
for i, (images, boxes, labels, _,
difficulties) in enumerate(tqdm(test_loader,
desc='Evaluating')):
# images = torch.stack(images, dim=0).to(config.device)
images = images.to(config.device)
boxes = [b.to(config.device) for b in boxes]
labels = [l.to(config.device) for l in labels]
difficulties = [d.to(config.device) for d in difficulties]
# Forward prop.
time_start = time.time()
outputs = model(images)
if len(outputs) == 2:
predicted_locs, predicted_scores = model(images)
prior_positives_idx = None
elif len(outputs) == 7:
_, _, _, _, predicted_locs, predicted_scores, prior_positives_idx = outputs
else:
raise NotImplementedError
if config['focal_type'].lower() == 'sigmoid':
det_boxes_batch, det_labels_batch, det_scores_batch = \
detect_focal(predicted_locs,
predicted_scores,
min_score=config.nms['min_score'],
max_overlap=config.nms['max_overlap'],
top_k=config.nms['top_k'], priors_cxcy=model.priors_cxcy,
config=config, prior_positives_idx=prior_positives_idx)
elif config['focal_type'].lower() == 'softmax':
det_boxes_batch, det_labels_batch, det_scores_batch = \
detect(predicted_locs,
predicted_scores,
min_score=config.nms['min_score'],
max_overlap=config.nms['max_overlap'],
top_k=config.nms['top_k'], priors_cxcy=model.priors_cxcy,
config=config, prior_positives_idx=prior_positives_idx)
else:
print('focal type should be either softmax or sigmoid.')
raise NotImplementedError
time_end = time.time()
det_boxes.extend(det_boxes_batch)
det_labels.extend(det_labels_batch)
det_scores.extend(det_scores_batch)
true_boxes.extend(boxes)
true_labels.extend(labels)
true_difficulties.extend(difficulties)
detect_speed.append((time_end - time_start) / len(labels))
# Calculate mAP
APs, mAP = calculate_mAP(det_boxes, det_labels, det_scores, true_boxes,
true_labels, true_difficulties,
config['eval_overlap'], config.label_map,
config.device)
# Print AP for each class
pp.pprint(APs)
str_print = 'EVAL: Mean Average Precision {0:.4f}, ' \
'avg speed {1:.3f} Hz, lr {2:.6f}'.format(mAP, 1. / np.mean(detect_speed),
config.scheduler.get_last_lr()[0])
config.logger.info(str_print)
del predicted_locs, predicted_scores, boxes, labels, images, difficulties
return APs, mAP