def submit_vis_images(engine):
aim_images = []
for outputs in engine.state.model_outputs:
for image, target, pred in zip(outputs["x"], outputs["y"],
outputs["y_pred"]):
image = (image * 255).byte()
pred_labels = [
Dataset.class2name[label.item()]
for label in pred["labels"]
]
pred_boxes = pred["boxes"].long()
image = draw_bounding_boxes(image,
pred_boxes,
pred_labels,
colors="red")
target_labels = [
Dataset.class2name[label.item()]
for label in target["labels"]
]
target_boxes = target["boxes"].long()
image = draw_bounding_boxes(image,
target_boxes,
target_labels,
colors="green")
aim_images.append(aim.Image(image.numpy().transpose(
(1, 2, 0))))
aim_logger.experiment.track(aim_images,
name="vis",
step=trainer.state.epoch)
def test_draw_boxes_warning():
img = torch.full((3, 100, 100), 255, dtype=torch.uint8)
with pytest.warns(
UserWarning,
match=re.escape(
"Argument 'font_size' will be ignored since 'font' is not set."
)):
utils.draw_bounding_boxes(img, boxes, font_size=11)
def test_draw_boxes_grayscale():
img = torch.full((1, 4, 4), fill_value=255, dtype=torch.uint8)
boxes = torch.tensor([[0, 0, 3, 3]], dtype=torch.int64)
bboxed_img = utils.draw_bounding_boxes(image=img,
boxes=boxes,
colors=["#1BBC9B"])
assert bboxed_img.size(0) == 3
def test_draw_boxes():
img = torch.full((3, 100, 100), 255, dtype=torch.uint8)
img_cp = img.clone()
boxes_cp = boxes.clone()
labels = ["a", "b", "c", "d"]
colors = ["green", "#FF00FF", (0, 255, 0), "red"]
result = utils.draw_bounding_boxes(img,
boxes,
labels=labels,
colors=colors,
fill=True)
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets",
"fakedata", "draw_boxes_util.png")
if not os.path.exists(path):
res = Image.fromarray(result.permute(1, 2, 0).contiguous().numpy())
res.save(path)
if PILLOW_VERSION >= (8, 2):
# The reference image is only valid for new PIL versions
expected = torch.as_tensor(np.array(Image.open(path))).permute(2, 0, 1)
assert_equal(result, expected)
# Check if modification is not in place
assert_equal(boxes, boxes_cp)
assert_equal(img, img_cp)
def render_frame(idx, frame, dets, tracks=False):
from torchvision.utils import draw_bounding_boxes
from ml.vision.utils import rgb, COLORS91
from ml.vision.datasets.coco import COCO80_CLASSES
if tracks:
tids, dets = list(zip(*dets))
dets = th.stack(dets)
labels = [f"[{int(c)}][{tid}]" for tid, c in zip(tids, dets[:, 5])]
colors = [rgb(tid, integral=True) for tid in tids]
annot_dict[idx]['tracks'] = {
tid: det
for tid, det in zip(tids, dets.tolist())
}
else:
labels = [COCO80_CLASSES[i] for i in dets[:, -1].int()]
colors = [COLORS91[i] for i in dets[:, 5].int()]
annot_dict[idx]['detections'] = dets.tolist()
# get boxes: x1, y1, x2, y2
boxes = dets[:, :4]
# draw bounding boxes
frame = draw_bounding_boxes(frame,
boxes=boxes,
labels=labels,
colors=colors,
fill=True,
width=3,
font_size=25)
return frame
def draw_bounding_box(self, bounding_box: BoundingBox,
**kwargs: Any) -> Image:
# TODO: this is useful for developing and debugging but we should remove or at least revisit this before we
# promote this out of the prototype state
return Image.new_like(
self,
draw_bounding_boxes(self,
bounding_box.to_format("xyxy").view(-1, 4),
**kwargs))
def test_draw_no_boxes():
img = torch.full((3, 100, 100), 0, dtype=torch.uint8)
boxes = torch.full((0, 4), 0, dtype=torch.float)
with pytest.warns(UserWarning,
match=re.escape(
"boxes doesn't contain any box. No box was drawn")):
res = utils.draw_bounding_boxes(img, boxes)
# Check that the function didnt change the image
assert res.eq(img).all()
def test_draw_invalid_boxes():
img_tp = ((1, 1, 1), (1, 2, 3))
img_wrong1 = torch.full((3, 5, 5), 255, dtype=torch.float)
img_wrong2 = torch.full((1, 3, 5, 5), 255, dtype=torch.uint8)
boxes = torch.tensor([[0, 0, 20, 20], [0, 0, 0, 0],
[10, 15, 30, 35], [23, 35, 93, 95]], dtype=torch.float)
with pytest.raises(TypeError, match="Tensor expected"):
utils.draw_bounding_boxes(img_tp, boxes)
with pytest.raises(ValueError, match="Tensor uint8 expected"):
utils.draw_bounding_boxes(img_wrong1, boxes)
with pytest.raises(ValueError, match="Pass individual images, not batches"):
utils.draw_bounding_boxes(img_wrong2, boxes)
with pytest.raises(ValueError, match="Only grayscale and RGB images are supported"):
utils.draw_bounding_boxes(img_wrong2[0][:2], boxes)
def test_draw_boxes(self):
img = torch.full((3, 100, 100), 255, dtype=torch.uint8)
boxes = torch.tensor([[0, 0, 20, 20], [0, 0, 0, 0],
[10, 15, 30, 35], [23, 35, 93, 95]], dtype=torch.float)
labels = ["a", "b", "c", "d"]
colors = ["green", "#FF00FF", (0, 255, 0), "red"]
result = utils.draw_bounding_boxes(img, boxes, labels=labels, colors=colors)
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets", "fakedata", "draw_boxes_util.png")
if not os.path.exists(path):
res = Image.fromarray(result.permute(1, 2, 0).contiguous().numpy())
res.save(path)
expected = torch.as_tensor(np.array(Image.open(path))).permute(2, 0, 1)
self.assertTrue(torch.equal(result, expected))
def test_draw_boxes_vanilla(self):
img = torch.full((3, 100, 100), 0, dtype=torch.uint8)
img_cp = img.clone()
boxes_cp = boxes.clone()
result = utils.draw_bounding_boxes(img, boxes, fill=False, width=7)
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets", "fakedata", "draw_boxes_vanilla.png")
if not os.path.exists(path):
res = Image.fromarray(result.permute(1, 2, 0).contiguous().numpy())
res.save(path)
expected = torch.as_tensor(np.array(Image.open(path))).permute(2, 0, 1)
self.assertTrue(torch.equal(result, expected))
# Check if modification is not in place
self.assertTrue(torch.all(torch.eq(boxes, boxes_cp)).item())
self.assertTrue(torch.all(torch.eq(img, img_cp)).item())
def test_draw_boxes(self):
img = torch.full((3, 100, 100), 255, dtype=torch.uint8)
img_cp = img.clone()
boxes_cp = boxes.clone()
labels = ["a", "b", "c", "d"]
colors = ["green", "#FF00FF", (0, 255, 0), "red"]
result = utils.draw_bounding_boxes(img, boxes, labels=labels, colors=colors, fill=True)
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets", "fakedata", "draw_boxes_util.png")
if not os.path.exists(path):
res = Image.fromarray(result.permute(1, 2, 0).contiguous().numpy())
res.save(path)
expected = torch.as_tensor(np.array(Image.open(path))).permute(2, 0, 1)
self.assertTrue(torch.equal(result, expected))
# Check if modification is not in place
self.assertTrue(torch.all(torch.eq(boxes, boxes_cp)).item())
self.assertTrue(torch.all(torch.eq(img, img_cp)).item())
def test_draw_boxes(self):
img = torch.full((3, 100, 100), 255, dtype=torch.uint8)
boxes = torch.tensor(
[[0, 0, 20, 20], [0, 0, 0, 0], [10, 15, 30, 35], [23, 35, 93, 95]],
dtype=torch.float)
labels = ["a", "b", "c", "d"]
colors = ["green", "#FF00FF", (0, 255, 0), "red"]
result = utils.draw_bounding_boxes(img,
boxes,
labels=labels,
colors=colors)
path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"assets", "fakedata", "draw_boxes_util.png")
if not os.path.exists(path):
write_png(result, path)
expected = read_image(path)
self.assertTrue(torch.equal(result, expected))
def visualize_batch(self):
if self.mode == "train":
loader = DataLoader(self,
batch_size=4,
shuffle=True,
num_workers=4,
collate_fn=self._collate_fn)
else:
loader = DataLoader(self,
batch_size=4,
shuffle=True,
num_workers=4,
collate_fn=self._collate_fn)
images, targets, image_ids = next(iter(loader))
bbx_ = []
for i, img in enumerate(images):
img_int = img * 255
img_int = img_int.type(torch.uint8)
bboxes_int = targets[i]['boxes'].type(torch.uint8)
bbclrs = []
bbclss = []
for jj, label in enumerate(targets[i]['labels']):
bbclrs.append(self.id_clr[int(label)])
bbclss.append(self._label_to_name(int(label)))
bbx_.append(
draw_bounding_boxes(img_int,
boxes=bboxes_int,
colors=bbclrs,
font_size=20,
labels=bbclss))
self._show(bbx_)
grid = make_grid([dog1_int, dog2_int, dog1_int, dog2_int])
show(grid)
####################################
# Visualizing bounding boxes
# --------------------------
# We can use :func:`~torchvision.utils.draw_bounding_boxes` to draw boxes on an
# image. We can set the colors, labels, width as well as font and font size.
# The boxes are in ``(xmin, ymin, xmax, ymax)`` format.
from torchvision.utils import draw_bounding_boxes
boxes = torch.tensor([[50, 50, 100, 200], [210, 150, 350, 430]],
dtype=torch.float)
colors = ["blue", "yellow"]
result = draw_bounding_boxes(dog1_int, boxes, colors=colors, width=5)
show(result)
#####################################
# Naturally, we can also plot bounding boxes produced by torchvision detection
# models. Here is demo with a Faster R-CNN model loaded from
# :func:`~torchvision.models.detection.fasterrcnn_resnet50_fpn`
# model. You can also try using a RetinaNet with
# :func:`~torchvision.models.detection.retinanet_resnet50_fpn`. For more details
# on the output of such models, you may refer to :ref:`instance_seg_output`.
from torchvision.models.detection import fasterrcnn_resnet50_fpn
from torchvision.transforms.functional import convert_image_dtype
batch_int = torch.stack([dog1_int, dog2_int])
batch = convert_image_dtype(batch_int, dtype=torch.float)
def test_draw_invalid_boxes():
img_tp = ((1, 1, 1), (1, 2, 3))
img_wrong1 = torch.full((3, 5, 5), 255, dtype=torch.float)
img_wrong2 = torch.full((1, 3, 5, 5), 255, dtype=torch.uint8)
img_correct = torch.zeros((3, 10, 10), dtype=torch.uint8)
boxes = torch.tensor(
[[0, 0, 20, 20], [0, 0, 0, 0], [10, 15, 30, 35], [23, 35, 93, 95]],
dtype=torch.float)
labels_wrong = ["one", "two"]
colors_wrong = ["pink", "blue"]
with pytest.raises(TypeError, match="Tensor expected"):
utils.draw_bounding_boxes(img_tp, boxes)
with pytest.raises(ValueError, match="Tensor uint8 expected"):
utils.draw_bounding_boxes(img_wrong1, boxes)
with pytest.raises(ValueError,
match="Pass individual images, not batches"):
utils.draw_bounding_boxes(img_wrong2, boxes)
with pytest.raises(ValueError,
match="Only grayscale and RGB images are supported"):
utils.draw_bounding_boxes(img_wrong2[0][:2], boxes)
with pytest.raises(ValueError, match="Number of boxes"):
utils.draw_bounding_boxes(img_correct, boxes, labels_wrong)
with pytest.raises(ValueError, match="Number of colors"):
utils.draw_bounding_boxes(img_correct, boxes, colors=colors_wrong)
def test_draw_boxes_colors(colors):
img = torch.full((3, 100, 100), 0, dtype=torch.uint8)
utils.draw_bounding_boxes(img, boxes, fill=False, width=7, colors=colors)
grid = make_grid([dog1_int, dog2_int, dog1_int, dog2_int])
show(grid)
####################################
# Visualizing bounding boxes
# --------------------------
# We can use :func:`~torchvision.utils.draw_bounding_boxes` to draw boxes on an
# image. We can set the colors, labels, width as well as font and font size.
# The boxes are in ``(xmin, ymin, xmax, ymax)`` format.
from torchvision.utils import draw_bounding_boxes
boxes = torch.tensor([[50, 50, 100, 200], [210, 150, 350, 430]],
dtype=torch.float)
colors = ["blue", "yellow"]
result = draw_bounding_boxes(dog1_int, boxes, colors=colors, width=5)
show(result)
#####################################
# Naturally, we can also plot bounding boxes produced by torchvision detection
# models. Here is demo with a Faster R-CNN model loaded from
# :func:`~torchvision.models.detection.fasterrcnn_resnet50_fpn`
# model. You can also try using a RetinaNet with
# :func:`~torchvision.models.detection.retinanet_resnet50_fpn`.
from torchvision.models.detection import fasterrcnn_resnet50_fpn
from torchvision.transforms.functional import convert_image_dtype
batch_int = torch.stack([dog1_int, dog2_int])
batch = convert_image_dtype(batch_int, dtype=torch.float)
# It returns the boxes in ``(xmin, ymin, xmax, ymax)`` format. from torchvision.ops import masks_to_boxes boxes = masks_to_boxes(masks) print(boxes.size()) print(boxes) #################################### # As the shape denotes, there are 3 boxes and in ``(xmin, ymin, xmax, ymax)`` format. # These can be visualized very easily with :func:`~torchvision.utils.draw_bounding_boxes` utility # provided in :ref:`torchvision.utils <utils>`. from torchvision.utils import draw_bounding_boxes drawn_boxes = draw_bounding_boxes(img, boxes, colors="red") show(drawn_boxes) ################################### # These boxes can now directly be used by detection models in torchvision. # Here is demo with a Faster R-CNN model loaded from # :func:`~torchvision.models.detection.fasterrcnn_resnet50_fpn` from torchvision.models.detection import fasterrcnn_resnet50_fpn, FasterRCNN_ResNet50_FPN_Weights weights = FasterRCNN_ResNet50_FPN_Weights.DEFAULT model = fasterrcnn_resnet50_fpn(weights=weights, progress=False) print(img.size()) tranforms = weights.transforms() img = tranforms(img)
