def _check_rpn_loss(self, xp): locs = [ chainer.Variable(_random_array(xp, (2, 32 * 32 * 3, 4))), chainer.Variable(_random_array(xp, (2, 16 * 16 * 3, 4))), chainer.Variable(_random_array(xp, (2, 8 * 8 * 3, 4))), ] confs = [ chainer.Variable(_random_array(xp, (2, 32 * 32 * 3))), chainer.Variable(_random_array(xp, (2, 16 * 16 * 3))), chainer.Variable(_random_array(xp, (2, 8 * 8 * 3))), ] anchors = RPN(scales=(1 / 2, 1 / 4, 1 / 8)) \ .anchors(((32, 32), (16, 16), (8, 8))) bboxes = [ xp.array(((2, 4, 6, 7), (1, 12, 3, 30)), dtype=np.float32), xp.array(((10, 2, 12, 12), ), dtype=np.float32), ] loc_loss, conf_loss = rpn_loss(locs, confs, anchors, ((480, 640), (320, 320)), bboxes) self.assertIsInstance(loc_loss, chainer.Variable) self.assertIsInstance(loc_loss.array, xp.ndarray) self.assertEqual(loc_loss.shape, ()) self.assertIsInstance(conf_loss, chainer.Variable) self.assertIsInstance(conf_loss.array, xp.ndarray) self.assertEqual(conf_loss.shape, ())
def forward(self, imgs, bboxes, labels): x, scales = self.model.prepare(imgs) bboxes = [ self.xp.array(bbox) * scale for bbox, scale in zip(bboxes, scales) ] labels = [self.xp.array(label) for label in labels] with chainer.using_config('train', False): hs = self.model.extractor(x) rpn_locs, rpn_confs = self.model.rpn(hs) anchors = self.model.rpn.anchors(h.shape[2:] for h in hs) rpn_loc_loss, rpn_conf_loss = rpn_loss( rpn_locs, rpn_confs, anchors, [(int(img.shape[1] * scale), int(img.shape[2] * scale)) for img, scale in zip(imgs, scales)], bboxes) rois, roi_indices = self.model.rpn.decode(rpn_locs, rpn_confs, anchors, x.shape) rois = self.xp.vstack([rois] + bboxes) roi_indices = self.xp.hstack([roi_indices] + [ self.xp.array((i, ) * len(bbox)) for i, bbox in enumerate(bboxes) ]) rois, roi_indices = self.model.head.distribute(rois, roi_indices) rois, roi_indices, head_gt_locs, head_gt_labels = head_loss_pre( rois, roi_indices, self.model.head.std, bboxes, labels) head_locs, head_confs = self.model.head(hs, rois, roi_indices) head_loc_loss, head_conf_loss = head_loss_post(head_locs, head_confs, roi_indices, head_gt_locs, head_gt_labels, len(x)) loss = rpn_loc_loss + rpn_conf_loss + head_loc_loss + head_conf_loss chainer.reporter.report( { 'loss': loss, 'loss/rpn/loc': rpn_loc_loss, 'loss/rpn/conf': rpn_conf_loss, 'loss/head/loc': head_loc_loss, 'loss/head/conf': head_conf_loss }, self) return loss
def forward(self, imgs, bboxes, labels, masks=None): if np.any([len(bbox) == 0 for bbox in bboxes]): return chainer.Variable(self.xp.zeros((), dtype=np.float32)) B = len(imgs) pad_size = np.array([im.shape[1:] for im in imgs]).max(axis=0) pad_size = (np.ceil(pad_size / self.model.stride) * self.model.stride).astype(int) x = np.zeros((len(imgs), 3, pad_size[0], pad_size[1]), dtype=np.float32) for i, img in enumerate(imgs): _, H, W = img.shape x[i, :, :H, :W] = img x = self.xp.array(x) bboxes = [self.xp.array(bbox) for bbox in bboxes] labels = [self.xp.array(label) for label in labels] sizes = [img.shape[1:] for img in imgs] with chainer.using_config("train", False): hs = self.model.extractor(x) rpn_locs, rpn_confs = self.model.rpn(hs) anchors = self.model.rpn.anchors(h.shape[2:] for h in hs) rpn_loc_loss, rpn_conf_loss = rpn_loss(rpn_locs, rpn_confs, anchors, sizes, bboxes) rois, roi_indices = self.model.rpn.decode(rpn_locs, rpn_confs, anchors, x.shape) rois = self.xp.vstack([rois] + bboxes) roi_indices = self.xp.hstack([roi_indices] + [ self.xp.array((i, ) * len(bbox)) for i, bbox in enumerate(bboxes) ]) rois, roi_indices = self.model.bbox_head.distribute(rois, roi_indices) rois, roi_indices, head_gt_locs, head_gt_labels = bbox_head_loss_pre( rois, roi_indices, self.model.bbox_head.std, bboxes, labels) head_locs, head_confs = self.model.bbox_head(hs, rois, roi_indices) head_loc_loss, head_conf_loss = bbox_head_loss_post( head_locs, head_confs, roi_indices, head_gt_locs, head_gt_labels, B) mask_loss = 0 if masks is not None: # For reducing unnecessary CPU/GPU copy, `masks` is kept in CPU. pad_masks = [ np.zeros((mask.shape[0], pad_size[0], pad_size[1]), dtype=np.bool) for mask in masks ] for i, mask in enumerate(masks): _, H, W = mask.shape pad_masks[i][:, :H, :W] = mask masks = pad_masks ( mask_rois, mask_roi_indices, gt_segms, gt_mask_labels, ) = mask_head_loss_pre( rois, roi_indices, masks, bboxes, head_gt_labels, self.model.mask_head.segm_size, ) n_roi = sum([len(roi) for roi in mask_rois]) if n_roi > 0: segms = self.model.mask_head(hs, mask_rois, mask_roi_indices) mask_loss = mask_head_loss_post(segms, mask_roi_indices, gt_segms, gt_mask_labels, B) else: # Compute dummy variables to complete the computational graph mask_rois[0] = self.xp.array([[0, 0, 1, 1]], dtype=np.float32) mask_roi_indices[0] = self.xp.array([0], dtype=np.int32) segms = self.model.mask_head(hs, mask_rois, mask_roi_indices) mask_loss = 0 * F.sum(segms) loss = (rpn_loc_loss + rpn_conf_loss + head_loc_loss + head_conf_loss + mask_loss) chainer.reporter.report( { "loss": loss, "loss/rpn/loc": rpn_loc_loss, "loss/rpn/conf": rpn_conf_loss, "loss/bbox_head/loc": head_loc_loss, "loss/bbox_head/conf": head_conf_loss, "loss/mask_head": mask_loss, }, self, ) return loss