def eval_dataset(config, net, loss_fn, loader, device, e_range='all'):
net.eval()
if config['mGPUs']:
net.module.set_decode(True)
else:
net.set_decode(True)
t_fwds = 0
t_post = 0
loss_sum = 0
img_list = range(len(loader.dataset))
if e_range != 'all':
e_range = min(e_range, len(loader.dataset))
img_list = random.sample(img_list, e_range)
log_img_list = random.sample(img_list, 10)
gts = 0
preds = 0
all_scores = []
all_matches = []
log_images = []
with torch.no_grad():
for image_id in img_list:
#tic = time.time()
num_gt, num_pred, scores, pred_image, pred_match, loss, t_forward, t_nms = \
eval_one(net, loss_fn, config, loader, image_id, device, plot=False)
gts += num_gt
preds += num_pred
loss_sum += loss
all_scores.extend(list(scores))
all_matches.extend(list(pred_match))
t_fwds += t_forward
t_post += t_nms
if image_id in log_img_list:
log_images.append(pred_image)
#print(time.time() - tic)
all_scores = np.array(all_scores)
all_matches = np.array(all_matches)
sort_ids = np.argsort(all_scores)
all_matches = all_matches[sort_ids[::-1]]
metrics = {}
AP, precisions, recalls, precision, recall = compute_ap(
all_matches, gts, preds)
metrics['AP'] = AP
metrics['Precision'] = precision
metrics['Recall'] = recall
metrics['loss'] = loss_sum / len(img_list)
metrics['Forward Pass Time'] = t_fwds / len(img_list)
metrics['Postprocess Time'] = t_post / len(img_list)
return metrics, precisions, recalls, log_images
def eval_batch(config, net, loss_fn, loader, device, eval_range='all'):
net.eval()
if config['mGPUs']:
net.module.set_decode(True)
else:
net.set_decode(True)
cls_loss = 0
loc_loss = 0
all_scores = []
all_matches = []
log_images = []
gts = 0
preds = 0
t_fwd = 0
t_nms = 0
log_img_list = random.sample(range(len(loader.dataset)), 10)
with torch.no_grad():
for i, data in enumerate(loader):
tic = time.time()
input, label_map, image_id = data
input = input.to(device)
label_map = label_map.to(device)
tac = time.time()
predictions = net(input)
t_fwd += time.time() - tac
loss, cls, loc = loss_fn(predictions, label_map)
cls_loss += cls
loc_loss += loc
t_fwd += (time.time() - tic)
toc = time.time()
# Parallel post-processing
predictions = list(torch.split(predictions.cpu(), 1, dim=0))
batch_size = len(predictions)
with Pool(processes=1) as pool:
preds_filtered = pool.starmap(filter_pred,
[(config, pred)
for pred in predictions])
t_nms += (time.time() - toc)
args = []
for j in range(batch_size):
_, label_list = loader.dataset.get_label(image_id[j].item())
corners, scores = preds_filtered[j]
gts += len(label_list)
preds += len(scores)
all_scores.extend(list(scores))
if image_id[j] in log_img_list:
input_np = input[j].cpu().permute(1, 2, 0).numpy()
pred_image = get_bev(input_np, corners)
log_images.append(pred_image)
arg = (np.array(label_list), corners, scores)
args.append(arg)
# Parallel compute matchesi
with Pool(processes=1) as pool:
matches = pool.starmap(compute_matches, args)
for j in range(batch_size):
all_matches.extend(list(matches[j][1]))
#print(time.time() -tic)
all_scores = np.array(all_scores)
all_matches = np.array(all_matches)
sort_ids = np.argsort(all_scores)
all_matches = all_matches[sort_ids[::-1]]
metrics = {}
AP, precisions, recalls, precision, recall = compute_ap(
all_matches, gts, preds)
metrics['AP'] = AP
metrics['Precision'] = precision
metrics['Recall'] = recall
metrics['Forward Pass Time'] = t_fwd / len(loader.dataset)
metrics['Postprocess Time'] = t_nms / len(loader.dataset)
cls_loss = cls_loss / len(loader)
loc_loss = loc_loss / len(loader)
metrics['loss'] = cls_loss + loc_loss
return metrics, precisions, recalls, log_images