def validation(net: STMask, valid_data=False, output_metrics_file=None):
cfg.mask_proto_debug = args.mask_proto_debug
if not valid_data:
cfg.valid_sub_dataset.test_mode = True
dataset = get_dataset(cfg.valid_sub_dataset)
else:
cfg.valid_dataset.test_mode = True
dataset = get_dataset(cfg.valid_dataset)
frame_times = MovingAverage()
dataset_size = math.ceil(len(dataset) /
args.batch_size) if args.max_images < 0 else min(
args.max_images, len(dataset))
progress_bar = ProgressBar(30, dataset_size)
print()
data_loader = data.DataLoader(dataset,
args.batch_size,
shuffle=False,
collate_fn=detection_collate,
pin_memory=True)
results = []
try:
# Main eval loop
for it, data_batch in enumerate(data_loader):
timer.reset()
with timer.env('Load Data'):
images, images_meta, ref_images, ref_images_meta = prepare_data(
data_batch, is_cuda=True, train_mode=False)
pad_h, pad_w = images.size()[2:4]
with timer.env('Network Extra'):
preds = net(images,
img_meta=images_meta,
ref_x=ref_images,
ref_imgs_meta=ref_images_meta)
if it == dataset_size - 1:
batch_size = len(dataset) % args.batch_size
else:
batch_size = images.size(0)
for batch_id in range(batch_size):
cfg.preserve_aspect_ratio = True
preds_cur = postprocess_ytbvis(
preds[batch_id],
pad_h,
pad_w,
images_meta[batch_id],
score_threshold=cfg.eval_conf_thresh)
segm_results = bbox2result_with_id(preds_cur, cfg.classes)
results.append(segm_results)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
if it > 1:
if batch_size == 0:
batch_size = 1
frame_times.add(timer.total_time() / batch_size)
if it > 1 and frame_times.get_avg() > 0:
fps = 1 / frame_times.get_avg()
else:
fps = 0
progress = (it + 1) / dataset_size * 100
progress_bar.set_val(it + 1)
print(
'\rProcessing Images %s %6d / %6d (%5.2f%%) %5.2f fps '
% (repr(progress_bar), it + 1, dataset_size, progress, fps),
end='')
print()
print('Dumping detections...')
if not valid_data:
results2json_videoseg(dataset, results, args.mask_det_file)
print('calculate evaluation metrics ...')
ann_file = cfg.valid_sub_dataset.ann_file
dt_file = args.mask_det_file
calc_metrics(ann_file, dt_file, output_file=output_metrics_file)
else:
results2json_videoseg(dataset, results,
output_metrics_file.replace('.txt', '.json'))
except KeyboardInterrupt:
print('Stopping...')
def evaluate(net: STMask, dataset):
net.detect.use_fast_nms = args.fast_nms
cfg.mask_proto_debug = args.mask_proto_debug
frame_times = MovingAverage()
dataset_size = math.ceil(len(dataset) /
args.batch_size) if args.max_images < 0 else min(
args.max_images, len(dataset))
progress_bar = ProgressBar(30, dataset_size)
print()
data_loader = data.DataLoader(dataset,
args.batch_size,
shuffle=False,
collate_fn=detection_collate,
pin_memory=True)
results = []
try:
# Main eval loop
for it, data_batch in enumerate(data_loader):
timer.reset()
with timer.env('Load Data'):
images, images_meta, ref_images, ref_images_meta = prepare_data(
data_batch, is_cuda=True, train_mode=False)
pad_h, pad_w = images.size()[2:4]
with timer.env('Network Extra'):
preds = net(images,
img_meta=images_meta,
ref_x=ref_images,
ref_imgs_meta=ref_images_meta)
# Perform the meat of the operation here depending on our mode.
if it == dataset_size - 1:
batch_size = len(dataset) % args.batch_size
else:
batch_size = images.size(0)
for batch_id in range(batch_size):
if args.display:
img_id = (images_meta[batch_id]['video_id'],
images_meta[batch_id]['frame_id'])
if not cfg.display_mask_single:
img_numpy = prep_display(
preds[batch_id],
images[batch_id],
pad_h,
pad_w,
img_meta=images_meta[batch_id],
img_ids=img_id)
else:
for p in range(
preds[batch_id]['detection']['box'].size(0)):
preds_single = {'detection': {}}
for k in preds[batch_id]['detection']:
if preds[batch_id]['detection'][
k] is not None and k not in {'proto'}:
preds_single['detection'][k] = preds[
batch_id]['detection'][k][p]
else:
preds_single['detection'][k] = None
preds_single['net'] = preds[batch_id]['net']
preds_single['detection'][
'box_ids'] = torch.tensor(-1)
img_numpy = prep_display(
preds_single,
images[batch_id],
pad_h,
pad_w,
img_meta=images_meta[batch_id],
img_ids=img_id)
plt.imshow(img_numpy)
plt.axis('off')
plt.savefig(''.join([
args.mask_det_file[:-12], 'out_single/',
str(img_id), '_',
str(p), '.png'
]))
plt.clf()
else:
cfg.preserve_aspect_ratio = True
preds_cur = postprocess_ytbvis(
preds[batch_id],
pad_h,
pad_w,
images_meta[batch_id],
score_threshold=cfg.eval_conf_thresh)
segm_results = bbox2result_with_id(preds_cur, cfg.classes)
results.append(segm_results)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
if it > 1:
frame_times.add(timer.total_time() / batch_size)
if args.display and not cfg.display_mask_single:
if it > 1:
print('Avg FPS: %.4f' % (1 / frame_times.get_avg()))
plt.imshow(img_numpy)
plt.axis('off')
plt.title(str(img_id))
root_dir = ''.join([
args.mask_det_file[:-12], 'out/',
str(images_meta[batch_id]['video_id']), '/'
])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
plt.savefig(''.join([
root_dir,
str(images_meta[batch_id]['frame_id']), '.png'
]))
plt.clf()
# plt.show()
elif not args.no_bar:
if it > 1: fps = 1 / frame_times.get_avg()
else: fps = 0
progress = (it + 1) / dataset_size * 100
progress_bar.set_val(it + 1)
print(
'\rProcessing Images %s %6d / %6d (%5.2f%%) %5.2f fps '
% (repr(progress_bar), it + 1, dataset_size, progress,
fps),
end='')
if not args.display and not args.benchmark:
print()
if args.output_json:
print('Dumping detections...')
results2json_videoseg(dataset, results, args.mask_det_file)
if cfg.use_valid_sub or cfg.use_train_sub:
if cfg.use_valid_sub:
print('calculate evaluation metrics ...')
ann_file = cfg.valid_sub_dataset.ann_file
else:
print('calculate train_sub metrics ...')
ann_file = cfg.train_dataset.ann_file
dt_file = args.mask_det_file
metrics = calc_metrics(ann_file, dt_file)
return metrics
elif args.benchmark:
print()
print()
print('Stats for the last frame:')
timer.print_stats()
avg_seconds = frame_times.get_avg()
print('Average: %5.2f fps, %5.2f ms' %
(1 / frame_times.get_avg(), 1000 * avg_seconds))
except KeyboardInterrupt:
print('Stopping...')
print('Loading model...', end='')
net = STMask()
net.load_weights(args.trained_model)
net.eval()
print(' Done.')
if args.cuda:
net = net.cuda()
if args.image is None:
if cfg.only_calc_metrics:
print('calculate evaluation metrics ...')
ann_file = cfg.valid_sub_dataset.ann_file
dt_file = args.mask_det_file
print('det_file:', dt_file)
metrics = calc_metrics(ann_file, dt_file)
metrics_name = [
'mAP', 'AP50', 'AP75', 'small', 'medium', 'large', 'AR1',
'AR10', 'AR100', 'AR100_small', 'AR100_medium',
'AR100_large'
]
log_dir = 'weights/temp/train_log'
writer = SummaryWriter(log_dir=log_dir,
comment='_scalars',
filename_suffix='VIS')
for i_m in range(len(metrics_name)):
writer.add_scalar('valid_metrics/' + metrics_name[i_m],
metrics[i_m], 1)
else:
evaluate(net, val_dataset)
else: