def evaluate(net: Yolact, dataset, train_mode=False):
net.detect.use_fast_nms = args.fast_nms
net.detect.use_cross_class_nms = args.cross_class_nms
cfg.mask_proto_debug = args.mask_proto_debug
# TODO Currently we do not support Fast Mask Re-scroing in evalimage, evalimages, and evalvideo
if args.image is not None:
if ':' in args.image:
inp, out = args.image.split(':')
evalimage(net, inp, out)
else:
evalimage(net, args.image)
return
elif args.images is not None:
inp, out = args.images.split(':')
evalimages(net, inp, out)
return
elif args.video is not None:
if ':' in args.video:
inp, out = args.video.split(':')
evalvideo(net, inp, out)
else:
evalvideo(net, args.video)
return
frame_times = MovingAverage()
dataset_size = len(dataset) if args.max_images < 0 else min(
args.max_images, len(dataset))
progress_bar = ProgressBar(30, dataset_size)
print()
if not args.display and not args.benchmark:
# For each class and iou, stores tuples (score, isPositive)
# Index ap_data[type][iouIdx][classIdx]
ap_data = {
'box': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds],
'mask': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds]
}
detections = Detections()
else:
timer.disable('Load Data')
dataset_indices = list(range(len(dataset)))
if args.shuffle:
random.shuffle(dataset_indices)
elif not args.no_sort:
# Do a deterministic shuffle based on the image ids
#
# I do this because on python 3.5 dictionary key order is *random*, while in 3.6 it's
# the order of insertion. That means on python 3.6, the images come in the order they are in
# in the annotations file. For some reason, the first images in the annotations file are
# the hardest. To combat this, I use a hard-coded hash function based on the image ids
# to shuffle the indices we use. That way, no matter what python version or how pycocotools
# handles the data, we get the same result every time.
hashed = [badhash(x) for x in dataset.ids]
dataset_indices.sort(key=lambda x: hashed[x])
# dataset_size=1000
dataset_indices = dataset_indices[:dataset_size]
try:
# Main eval loop
dataset.batch_size = 1
dataset.num_workers = 1
for it, batch in enumerate(dataset):
timer.reset()
image_idx, img, gt, gt_masks, h, w, num_crowd = batch[0]
if not args.benchmark:
gt = gt.numpy()
gt_masks = gt_masks.numpy()
batch = img.reshape(1, img.shape[0], img.shape[1], img.shape[2])
# batch = jt.array([img])
with timer.env('Network Extra'):
preds = net(batch)
if args.display:
img_numpy = prep_display(preds, img, h, w)
elif args.benchmark:
prep_benchmark(preds, h, w)
else:
prep_metrics(ap_data, preds, img, gt, gt_masks, h, w,
num_crowd, dataset.ids[image_idx], detections)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
if it > 1:
frame_times.add(timer.total_time())
if args.display:
if it > 1:
print('Avg FPS: %.4f' % (1 / frame_times.get_avg()))
plt.imshow(img_numpy)
plt.title(str(dataset.ids[image_idx]))
plt.show()
elif not args.no_bar:
if it > 1: fps = 1 / frame_times.get_avg()
else: fps = 0
progress = (it + 1) / dataset_size * 100
progress_bar.set_val(it + 1)
print(
'\rProcessing Images %s %6d / %6d (%5.2f%%) %5.2f fps '
%
(repr(progress_bar), it + 1, dataset_size, progress, fps),
end='')
jt.sync_all(True)
if not args.display and not args.benchmark:
print()
if args.output_coco_json:
print('Dumping detections..')
if args.output_web_json:
detections.dump_web()
else:
detections.dump()
else:
if not train_mode:
print('Saving data..')
with open(args.ap_data_file, 'wb') as f:
pickle.dump(ap_data, f)
return calc_map(ap_data)
elif args.benchmark:
print()
print()
print('Stats for the last frame:')
timer.print_stats()
avg_seconds = frame_times.get_avg()
print('Average: %5.2f fps, %5.2f ms' %
(1 / frame_times.get_avg(), 1000 * avg_seconds))
except KeyboardInterrupt:
print('Stopping..')
net = net
# cudnn.benchmark = True
torch.set_default_tensor_type('torch.FloatTensor')
x = torch.zeros((1, 3, cfg.max_size, cfg.max_size))
y = net(x)
for p in net.prediction_layers:
print(p.last_conv_size)
print()
for k, a in y.items():
print(k + ': ', a.size(), torch.sum(a))
exit()
net(x)
# timer.disable('pass2')
avg = MovingAverage()
try:
while True:
timer.reset()
with timer.env('everything else'):
net(x)
avg.add(timer.total_time())
print('\033[2J') # Moves console cursor to 0,0
timer.print_stats()
print('Avg fps: %.2f\tAvg ms: %.2f ' %
(1 / avg.get_avg(), avg.get_avg() * 1000))
except KeyboardInterrupt:
pass
def evaluate(net: STMask, dataset):
net.detect.use_fast_nms = args.fast_nms
cfg.mask_proto_debug = args.mask_proto_debug
frame_times = MovingAverage()
dataset_size = math.ceil(len(dataset) /
args.batch_size) if args.max_images < 0 else min(
args.max_images, len(dataset))
progress_bar = ProgressBar(30, dataset_size)
print()
data_loader = data.DataLoader(dataset,
args.batch_size,
shuffle=False,
collate_fn=detection_collate,
pin_memory=True)
results = []
try:
# Main eval loop
for it, data_batch in enumerate(data_loader):
timer.reset()
with timer.env('Load Data'):
images, images_meta, ref_images, ref_images_meta = prepare_data(
data_batch, is_cuda=True, train_mode=False)
pad_h, pad_w = images.size()[2:4]
with timer.env('Network Extra'):
preds = net(images,
img_meta=images_meta,
ref_x=ref_images,
ref_imgs_meta=ref_images_meta)
# Perform the meat of the operation here depending on our mode.
if it == dataset_size - 1:
batch_size = len(dataset) % args.batch_size
else:
batch_size = images.size(0)
for batch_id in range(batch_size):
if args.display:
img_id = (images_meta[batch_id]['video_id'],
images_meta[batch_id]['frame_id'])
if not cfg.display_mask_single:
img_numpy = prep_display(
preds[batch_id],
images[batch_id],
pad_h,
pad_w,
img_meta=images_meta[batch_id],
img_ids=img_id)
else:
for p in range(
preds[batch_id]['detection']['box'].size(0)):
preds_single = {'detection': {}}
for k in preds[batch_id]['detection']:
if preds[batch_id]['detection'][
k] is not None and k not in {'proto'}:
preds_single['detection'][k] = preds[
batch_id]['detection'][k][p]
else:
preds_single['detection'][k] = None
preds_single['net'] = preds[batch_id]['net']
preds_single['detection'][
'box_ids'] = torch.tensor(-1)
img_numpy = prep_display(
preds_single,
images[batch_id],
pad_h,
pad_w,
img_meta=images_meta[batch_id],
img_ids=img_id)
plt.imshow(img_numpy)
plt.axis('off')
plt.savefig(''.join([
args.mask_det_file[:-12], 'out_single/',
str(img_id), '_',
str(p), '.png'
]))
plt.clf()
else:
cfg.preserve_aspect_ratio = True
preds_cur = postprocess_ytbvis(
preds[batch_id],
pad_h,
pad_w,
images_meta[batch_id],
score_threshold=cfg.eval_conf_thresh)
segm_results = bbox2result_with_id(preds_cur, cfg.classes)
results.append(segm_results)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
if it > 1:
frame_times.add(timer.total_time() / batch_size)
if args.display and not cfg.display_mask_single:
if it > 1:
print('Avg FPS: %.4f' % (1 / frame_times.get_avg()))
plt.imshow(img_numpy)
plt.axis('off')
plt.title(str(img_id))
root_dir = ''.join([
args.mask_det_file[:-12], 'out/',
str(images_meta[batch_id]['video_id']), '/'
])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
plt.savefig(''.join([
root_dir,
str(images_meta[batch_id]['frame_id']), '.png'
]))
plt.clf()
# plt.show()
elif not args.no_bar:
if it > 1: fps = 1 / frame_times.get_avg()
else: fps = 0
progress = (it + 1) / dataset_size * 100
progress_bar.set_val(it + 1)
print(
'\rProcessing Images %s %6d / %6d (%5.2f%%) %5.2f fps '
% (repr(progress_bar), it + 1, dataset_size, progress,
fps),
end='')
if not args.display and not args.benchmark:
print()
if args.output_json:
print('Dumping detections...')
results2json_videoseg(dataset, results, args.mask_det_file)
if cfg.use_valid_sub or cfg.use_train_sub:
if cfg.use_valid_sub:
print('calculate evaluation metrics ...')
ann_file = cfg.valid_sub_dataset.ann_file
else:
print('calculate train_sub metrics ...')
ann_file = cfg.train_dataset.ann_file
dt_file = args.mask_det_file
metrics = calc_metrics(ann_file, dt_file)
return metrics
elif args.benchmark:
print()
print()
print('Stats for the last frame:')
timer.print_stats()
avg_seconds = frame_times.get_avg()
print('Average: %5.2f fps, %5.2f ms' %
(1 / frame_times.get_avg(), 1000 * avg_seconds))
except KeyboardInterrupt:
print('Stopping...')
def evaluate(net: Yolact, dataset, train_mode=False):
net.detect.use_fast_nms = args.fast_nms
cfg.mask_proto_debug = args.mask_proto_debug
if args.image is not None:
if ':' in args.image:
inp, out = args.image.split(':')
evalimage(net, inp, out)
else:
evalimage(net, args.image)
return
elif args.images is not None:
inp, out = args.images.split(':')
evalimages(net, inp, out)
return
elif args.video is not None:
if ':' in args.video:
inp, out = args.video.split(':')
savevideo(net, inp, out)
else:
evalvideo(net, args.video)
return
frame_times = MovingAverage()
dataset_size = len(dataset) if args.max_images < 0 else min(
args.max_images, len(dataset))
progress_bar = ProgressBar(30, dataset_size)
print()
if not args.display and not args.benchmark:
# For each class and iou, stores tuples (score, isPositive)
# Index ap_data[type][iouIdx][classIdx]
ap_data = {
'box': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds],
'mask': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds]
}
detections = Detections()
else:
timer.disable('Load Data')
dataset_indices = list(range(len(dataset)))
if args.shuffle:
random.shuffle(dataset_indices)
elif not args.no_sort:
hashed = [badhash(x) for x in dataset.ids]
dataset_indices.sort(key=lambda x: hashed[x])
dataset_indices = dataset_indices[:dataset_size]
try:
# Main eval loop
for it, image_idx in enumerate(dataset_indices):
timer.reset()
with timer.env('Load Data'):
img, gt, gt_masks, h, w, num_crowd = dataset.pull_item(
image_idx)
# Test flag, do not upvote
if cfg.mask_proto_debug:
with open('scripts/info.txt', 'w') as f:
f.write(str(dataset.ids[image_idx]))
np.save('scripts/gt.npy', gt_masks)
batch = Variable(img.unsqueeze(0))
if args.cuda:
batch = batch.cuda()
with timer.env('Network Extra'):
preds = net(batch)
# Perform the meat of the operation here depending on our mode.
if args.display:
img_numpy = prep_display(preds, img, h, w)
elif args.benchmark:
prep_benchmark(preds, h, w)
else:
prep_metrics(ap_data, preds, img, gt, gt_masks, h, w,
num_crowd, dataset.ids[image_idx], detections)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
if it > 1:
frame_times.add(timer.total_time())
if args.display:
if it > 1:
print('Avg FPS: %.4f' % (1 / frame_times.get_avg()))
plt.imshow(img_numpy)
plt.title(str(dataset.ids[image_idx]))
plt.show()
elif not args.no_bar:
if it > 1: fps = 1 / frame_times.get_avg()
else: fps = 0
progress = (it + 1) / dataset_size * 100
progress_bar.set_val(it + 1)
print(
'\rProcessing Images %s %6d / %6d (%5.2f%%) %5.2f fps '
%
(repr(progress_bar), it + 1, dataset_size, progress, fps),
end='')
if not args.display and not args.benchmark:
print()
if args.output_coco_json:
print('Dumping detections...')
if args.output_web_json:
detections.dump_web()
else:
detections.dump()
else:
if not train_mode:
print('Saving data...')
with open(args.ap_data_file, 'wb') as f:
pickle.dump(ap_data, f)
return calc_map(ap_data)
elif args.benchmark:
print()
print()
print('Stats for the last frame:')
timer.print_stats()
avg_seconds = frame_times.get_avg()
print('Average: %5.2f fps, %5.2f ms' %
(1 / frame_times.get_avg(), 1000 * avg_seconds))
except KeyboardInterrupt:
print('Stopping...')
def evaluate(net: Yolact, dataset, train_mode=False):
net.detect.use_fast_nms = args.fast_nms
net.detect.use_cross_class_nms = args.cross_class_nms
cfg.mask_proto_debug = args.mask_proto_debug
# TODO Currently we do not support Fast Mask Re-scroing in evalimage, evalimages, and evalvideo
if args.image is not None:
if ":" in args.image:
inp, out = args.image.split(":")
evalimage(net, inp, out)
else:
evalimage(net, args.image)
return
elif args.images is not None:
inp, out = args.images.split(":")
evalimages(net, inp, out)
return
elif args.video is not None:
if ":" in args.video:
inp, out = args.video.split(":")
evalvideo(net, inp, out)
else:
evalvideo(net, args.video)
return
frame_times = MovingAverage()
dataset_size = (
len(dataset) if args.max_images < 0 else min(args.max_images, len(dataset))
)
progress_bar = ProgressBar(30, dataset_size)
print()
if not args.display and not args.benchmark:
# For each class and iou, stores tuples (score, isPositive)
# Index ap_data[type][iouIdx][classIdx]
ap_data = {
"box": [
[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds
],
"mask": [
[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds
],
}
detections = Detections()
else:
timer.disable("Load Data")
dataset_indices = list(range(len(dataset)))
if args.shuffle:
random.shuffle(dataset_indices)
elif not args.no_sort:
# Do a deterministic shuffle based on the image ids
#
# I do this because on python 3.5 dictionary key order is *random*, while in 3.6 it's
# the order of insertion. That means on python 3.6, the images come in the order they are in
# in the annotations file. For some reason, the first images in the annotations file are
# the hardest. To combat this, I use a hard-coded hash function based on the image ids
# to shuffle the indices we use. That way, no matter what python version or how pycocotools
# handles the data, we get the same result every time.
hashed = [badhash(x) for x in dataset.ids]
dataset_indices.sort(key=lambda x: hashed[x])
dataset_indices = dataset_indices[:dataset_size]
try:
# Main eval loop
for it, image_idx in enumerate(dataset_indices):
timer.reset()
with timer.env("Load Data"):
img, gt, gt_masks, h, w, num_crowd = dataset.pull_item(image_idx)
# Test flag, do not upvote
if cfg.mask_proto_debug:
with open("scripts/info.txt", "w") as f:
f.write(str(dataset.ids[image_idx]))
np.save("scripts/gt.npy", gt_masks)
batch = Variable(img.unsqueeze(0))
if args.cuda:
batch = batch.cuda()
with timer.env("Network Extra"):
preds = net(batch)
# Perform the meat of the operation here depending on our mode.
if args.display:
img_numpy = prep_display(preds, img, h, w)
elif args.benchmark:
prep_benchmark(preds, h, w)
else:
prep_metrics(
ap_data,
preds,
img,
gt,
gt_masks,
h,
w,
num_crowd,
dataset.ids[image_idx],
detections,
)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
if it > 1:
frame_times.add(timer.total_time())
if args.display:
if it > 1:
print("Avg FPS: %.4f" % (1 / frame_times.get_avg()))
plt.imshow(img_numpy)
plt.title(str(dataset.ids[image_idx]))
plt.show()
elif not args.no_bar:
if it > 1:
fps = 1 / frame_times.get_avg()
else:
fps = 0
progress = (it + 1) / dataset_size * 100
progress_bar.set_val(it + 1)
print(
"\rProcessing Images %s %6d / %6d (%5.2f%%) %5.2f fps "
% (repr(progress_bar), it + 1, dataset_size, progress, fps),
end="",
)
if not args.display and not args.benchmark:
print()
if args.output_coco_json:
print("Dumping detections...")
if args.output_web_json:
detections.dump_web()
else:
detections.dump()
else:
if not train_mode:
print("Saving data...")
with open(args.ap_data_file, "wb") as f:
pickle.dump(ap_data, f)
return calc_map(ap_data)
elif args.benchmark:
print()
print()
print("Stats for the last frame:")
timer.print_stats()
avg_seconds = frame_times.get_avg()
print(
"Average: %5.2f fps, %5.2f ms"
% (1 / frame_times.get_avg(), 1000 * avg_seconds)
)
except KeyboardInterrupt:
print("Stopping...")
def evaluate(net,
dataset,
max_num=-1,
during_training=False,
benchmark=False,
cocoapi=False,
traditional_nms=False):
frame_times = MovingAverage()
dataset_size = len(dataset) if max_num < 0 else min(max_num, len(dataset))
dataset_indices = list(range(len(dataset)))
dataset_indices = dataset_indices[:dataset_size]
progress_bar = ProgressBar(40, dataset_size)
if benchmark:
timer.disable('Data loading')
else:
# For each class and iou, stores tuples (score, isPositive)
# Index ap_data[type][iouIdx][classIdx]
ap_data = {
'box': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds],
'mask': [[APDataObject() for _ in cfg.dataset.class_names]
for _ in iou_thresholds]
}
make_json = Make_json()
for i, image_idx in enumerate(dataset_indices):
timer.reset()
with timer.env('Data loading'):
img, gt, gt_masks, h, w, num_crowd = dataset.pull_item(image_idx)
batch = Variable(img.unsqueeze(0))
if cuda:
batch = batch.cuda()
with timer.env('Network forward'):
net_outs = net(batch)
nms_outs = NMS(net_outs, traditional_nms)
if benchmark:
prep_benchmark(nms_outs, h, w)
else:
prep_metrics(ap_data, nms_outs, gt, gt_masks, h, w, num_crowd,
dataset.ids[image_idx], make_json, cocoapi)
# First couple of images take longer because we're constructing the graph.
# Since that's technically initialization, don't include those in the FPS calculations.
fps = 0
if i > 1 and not during_training:
frame_times.add(timer.total_time())
fps = 1 / frame_times.get_avg()
progress = (i + 1) / dataset_size * 100
progress_bar.set_val(i + 1)
print('\rProcessing: %s %d / %d (%.2f%%) %.2f fps ' %
(repr(progress_bar), i + 1, dataset_size, progress, fps),
end='')
if benchmark:
print('\n\nStats for the last frame:')
timer.print_stats()
avg_seconds = frame_times.get_avg()
print('Average: %5.2f fps, %5.2f ms' %
(1 / frame_times.get_avg(), 1000 * avg_seconds))
else:
if cocoapi:
make_json.dump()
print(f'\nJson files dumped, saved in: {json_path}.')
return
table = calc_map(ap_data)
print(table)
return table
