def testing(epo, model, test_loader):
model.eval()
conf_total = np.zeros((n_class, n_class))
with torch.no_grad():
for it, (images, labels, names) in enumerate(test_loader):
images = Variable(images)
labels = Variable(labels)
if args.gpu >= 0:
images = images.cuda(args.gpu)
labels = labels.cuda(args.gpu)
logits = model(images)
label = labels.cpu().numpy().squeeze().flatten()
prediction = logits.argmax(1).cpu().numpy().squeeze().flatten() # prediction and label are both 1-d array, size: minibatch*640*480
conf = confusion_matrix(label, prediction, [0,1,2,3,4,5,6,7,8]) # conf is n_class*n_class matrix, vertical axis: groundtruth, horizontal axis: prediction
conf_total += conf
print('|- %s, epo %s/%s. testing iter %s/%s.' % (args.model_name, epo, args.epoch_max, it+1, test_loader.n_iter))
precision, recall, IoU, = compute_results(conf_total)
writer.add_scalar('Test/average_precision',precision.mean(), epo)
writer.add_scalar('Test/average_recall', recall.mean(), epo)
writer.add_scalar('Test/average_IoU', IoU.mean(), epo)
return [np.mean(np.nan_to_num(precision)), np.mean(np.nan_to_num(recall)), np.mean(np.nan_to_num(IoU))]
def testing(epo, model, test_loader):
model.eval()
conf_total = np.zeros((args.n_class, args.n_class))
label_list = ["unlabeled", "car", "person", "bike", "curve", "car_stop", "guardrail", "color_cone", "bump"]
testing_results_file = os.path.join(weight_dir, 'testing_results_file.txt')
with torch.no_grad():
for it, (images, labels, names) in enumerate(test_loader):
images = Variable(images).cuda(args.gpu)
labels = Variable(labels).cuda(args.gpu)
logits = model(images)
label = labels.cpu().numpy().squeeze().flatten()
prediction = logits.argmax(1).cpu().numpy().squeeze().flatten() # prediction and label are both 1-d array, size: minibatch*640*480
conf = confusion_matrix(y_true=label, y_pred=prediction, labels=[0,1,2,3,4,5,6,7,8]) # conf is args.n_class*args.n_class matrix, vertical axis: groundtruth, horizontal axis: prediction
conf_total += conf
print('Test: %s, epo %s/%s, iter %s/%s, time %s' % (args.model_name, epo, args.epoch_max, it+1, len(test_loader),
datetime.datetime.now().replace(microsecond=0)-start_datetime))
precision, recall, IoU = compute_results(conf_total)
writer.add_scalar('Test/average_precision',precision.mean(), epo)
writer.add_scalar('Test/average_recall', recall.mean(), epo)
writer.add_scalar('Test/average_IoU', IoU.mean(), epo)
for i in range(len(precision)):
writer.add_scalar("Test(class)/precision_class_%s" % label_list[i], precision[i], epo)
writer.add_scalar("Test(class)/recall_class_%s"% label_list[i], recall[i],epo)
writer.add_scalar('Test(class)/Iou_%s'% label_list[i], IoU[i], epo)
if epo==0:
with open(testing_results_file, 'w') as f:
f.write("# %s, initial lr: %s, batch size: %s, date: %s \n" %(args.model_name, args.lr_start, args.batch_size, datetime.date.today()))
f.write("# epoch: unlabeled, car, person, bike, curve, car_stop, guardrail, color_cone, bump, average(nan_to_num). (Acc %, IoU %)\n")
with open(testing_results_file, 'a') as f:
f.write(str(epo)+': ')
for i in range(len(precision)):
f.write('%0.4f, %0.4f, ' % (100*recall[i], 100*IoU[i]))
f.write('%0.4f, %0.4f\n' % (100*np.mean(np.nan_to_num(recall)), 100*np.mean(np.nan_to_num(IoU))))
print('saving testing results.')
with open(testing_results_file, "r") as file:
writer.add_text('testing_results', file.read().replace('\n', ' \n'), epo)
logits = model(images) # logits.size(): mini_batch*num_class*480*640
end_time = time.time()
if it>=5: # # ignore the first 5 frames
ave_time_cost += (end_time-start_time)
# convert tensor to numpy 1d array
label = labels.cpu().numpy().squeeze().flatten()
prediction = logits.argmax(1).cpu().numpy().squeeze().flatten() # prediction and label are both 1-d array, size: minibatch*640*480
# generate confusion matrix frame-by-frame
conf = confusion_matrix(y_true=label, y_pred=prediction, labels=[0,1,2,3,4,5,6,7,8]) # conf is an n_class*n_class matrix, vertical axis: groundtruth, horizontal axis: prediction
conf_total += conf
# save demo images
visualize(image_name=names, predictions=logits.argmax(1), weight_name=args.weight_name)
print("%s, %s, frame %d/%d, %s, time cost: %.2f ms, demo result saved."
%(args.model_name, args.weight_name, it+1, len(test_loader), names, (end_time-start_time)*1000))
precision_per_class, recall_per_class, iou_per_class = compute_results(conf_total)
conf_total_matfile = os.path.join("./runs", 'conf_'+args.weight_name+'.mat')
savemat(conf_total_matfile, {'conf': conf_total}) # 'conf' is the variable name when loaded in Matlab
print('\n###########################################################################')
print('\n%s: %s test results (with batch size %d) on %s using %s:' %(args.model_name, args.weight_name, batch_size, datetime.date.today(), torch.cuda.get_device_name(args.gpu)))
print('\n* the tested dataset name: %s' % args.dataset_split)
print('* the tested image count: %d' % len(test_loader))
print('* the tested image size: %d*%d' %(args.img_height, args.img_width))
print('* the weight name: %s' %args.weight_name)
print('* the file name: %s' %args.file_name)
print("* recall per class: \n unlabeled: %.6f, car: %.6f, person: %.6f, bike: %.6f, curve: %.6f, car_stop: %.6f, guardrail: %.6f, color_cone: %.6f, bump: %.6f" \
%(recall_per_class[0], recall_per_class[1], recall_per_class[2], recall_per_class[3], recall_per_class[4], recall_per_class[5], recall_per_class[6], recall_per_class[7], recall_per_class[8]))
print("* iou per class: \n unlabeled: %.6f, car: %.6f, person: %.6f, bike: %.6f, curve: %.6f, car_stop: %.6f, guardrail: %.6f, color_cone: %.6f, bump: %.6f" \
%(iou_per_class[0], iou_per_class[1], iou_per_class[2], iou_per_class[3], iou_per_class[4], iou_per_class[5], iou_per_class[6], iou_per_class[7], iou_per_class[8]))
print("\n* average values (np.mean(x)): \n recall: %.6f, iou: %.6f" \