def _check_rpn_loss(self, xp):
locs = [
chainer.Variable(_random_array(xp, (2, 32 * 32 * 3, 4))),
chainer.Variable(_random_array(xp, (2, 16 * 16 * 3, 4))),
chainer.Variable(_random_array(xp, (2, 8 * 8 * 3, 4))),
]
confs = [
chainer.Variable(_random_array(xp, (2, 32 * 32 * 3))),
chainer.Variable(_random_array(xp, (2, 16 * 16 * 3))),
chainer.Variable(_random_array(xp, (2, 8 * 8 * 3))),
]
anchors = RPN(scales=(1 / 2, 1 / 4, 1 / 8)) \
.anchors(((32, 32), (16, 16), (8, 8)))
bboxes = [
xp.array(((2, 4, 6, 7), (1, 12, 3, 30)), dtype=np.float32),
xp.array(((10, 2, 12, 12), ), dtype=np.float32),
]
loc_loss, conf_loss = rpn_loss(locs, confs, anchors,
((480, 640), (320, 320)), bboxes)
self.assertIsInstance(loc_loss, chainer.Variable)
self.assertIsInstance(loc_loss.array, xp.ndarray)
self.assertEqual(loc_loss.shape, ())
self.assertIsInstance(conf_loss, chainer.Variable)
self.assertIsInstance(conf_loss.array, xp.ndarray)
self.assertEqual(conf_loss.shape, ())
def forward(self, imgs, bboxes, labels):
x, scales = self.model.prepare(imgs)
bboxes = [
self.xp.array(bbox) * scale for bbox, scale in zip(bboxes, scales)
]
labels = [self.xp.array(label) for label in labels]
with chainer.using_config('train', False):
hs = self.model.extractor(x)
rpn_locs, rpn_confs = self.model.rpn(hs)
anchors = self.model.rpn.anchors(h.shape[2:] for h in hs)
rpn_loc_loss, rpn_conf_loss = rpn_loss(
rpn_locs, rpn_confs, anchors,
[(int(img.shape[1] * scale), int(img.shape[2] * scale))
for img, scale in zip(imgs, scales)], bboxes)
rois, roi_indices = self.model.rpn.decode(rpn_locs, rpn_confs, anchors,
x.shape)
rois = self.xp.vstack([rois] + bboxes)
roi_indices = self.xp.hstack([roi_indices] + [
self.xp.array((i, ) * len(bbox)) for i, bbox in enumerate(bboxes)
])
rois, roi_indices = self.model.head.distribute(rois, roi_indices)
rois, roi_indices, head_gt_locs, head_gt_labels = head_loss_pre(
rois, roi_indices, self.model.head.std, bboxes, labels)
head_locs, head_confs = self.model.head(hs, rois, roi_indices)
head_loc_loss, head_conf_loss = head_loss_post(head_locs, head_confs,
roi_indices,
head_gt_locs,
head_gt_labels, len(x))
loss = rpn_loc_loss + rpn_conf_loss + head_loc_loss + head_conf_loss
chainer.reporter.report(
{
'loss': loss,
'loss/rpn/loc': rpn_loc_loss,
'loss/rpn/conf': rpn_conf_loss,
'loss/head/loc': head_loc_loss,
'loss/head/conf': head_conf_loss
}, self)
return loss
def forward(self, imgs, bboxes, labels, masks=None):
if np.any([len(bbox) == 0 for bbox in bboxes]):
return chainer.Variable(self.xp.zeros((), dtype=np.float32))
B = len(imgs)
pad_size = np.array([im.shape[1:] for im in imgs]).max(axis=0)
pad_size = (np.ceil(pad_size / self.model.stride) *
self.model.stride).astype(int)
x = np.zeros((len(imgs), 3, pad_size[0], pad_size[1]),
dtype=np.float32)
for i, img in enumerate(imgs):
_, H, W = img.shape
x[i, :, :H, :W] = img
x = self.xp.array(x)
bboxes = [self.xp.array(bbox) for bbox in bboxes]
labels = [self.xp.array(label) for label in labels]
sizes = [img.shape[1:] for img in imgs]
with chainer.using_config("train", False):
hs = self.model.extractor(x)
rpn_locs, rpn_confs = self.model.rpn(hs)
anchors = self.model.rpn.anchors(h.shape[2:] for h in hs)
rpn_loc_loss, rpn_conf_loss = rpn_loss(rpn_locs, rpn_confs, anchors,
sizes, bboxes)
rois, roi_indices = self.model.rpn.decode(rpn_locs, rpn_confs, anchors,
x.shape)
rois = self.xp.vstack([rois] + bboxes)
roi_indices = self.xp.hstack([roi_indices] + [
self.xp.array((i, ) * len(bbox)) for i, bbox in enumerate(bboxes)
])
rois, roi_indices = self.model.bbox_head.distribute(rois, roi_indices)
rois, roi_indices, head_gt_locs, head_gt_labels = bbox_head_loss_pre(
rois, roi_indices, self.model.bbox_head.std, bboxes, labels)
head_locs, head_confs = self.model.bbox_head(hs, rois, roi_indices)
head_loc_loss, head_conf_loss = bbox_head_loss_post(
head_locs, head_confs, roi_indices, head_gt_locs, head_gt_labels,
B)
mask_loss = 0
if masks is not None:
# For reducing unnecessary CPU/GPU copy, `masks` is kept in CPU.
pad_masks = [
np.zeros((mask.shape[0], pad_size[0], pad_size[1]),
dtype=np.bool) for mask in masks
]
for i, mask in enumerate(masks):
_, H, W = mask.shape
pad_masks[i][:, :H, :W] = mask
masks = pad_masks
(
mask_rois,
mask_roi_indices,
gt_segms,
gt_mask_labels,
) = mask_head_loss_pre(
rois,
roi_indices,
masks,
bboxes,
head_gt_labels,
self.model.mask_head.segm_size,
)
n_roi = sum([len(roi) for roi in mask_rois])
if n_roi > 0:
segms = self.model.mask_head(hs, mask_rois, mask_roi_indices)
mask_loss = mask_head_loss_post(segms, mask_roi_indices,
gt_segms, gt_mask_labels, B)
else:
# Compute dummy variables to complete the computational graph
mask_rois[0] = self.xp.array([[0, 0, 1, 1]], dtype=np.float32)
mask_roi_indices[0] = self.xp.array([0], dtype=np.int32)
segms = self.model.mask_head(hs, mask_rois, mask_roi_indices)
mask_loss = 0 * F.sum(segms)
loss = (rpn_loc_loss + rpn_conf_loss + head_loc_loss + head_conf_loss +
mask_loss)
chainer.reporter.report(
{
"loss": loss,
"loss/rpn/loc": rpn_loc_loss,
"loss/rpn/conf": rpn_conf_loss,
"loss/bbox_head/loc": head_loc_loss,
"loss/bbox_head/conf": head_conf_loss,
"loss/mask_head": mask_loss,
},
self,
)
return loss