Exemple #1
0
    def fetch_masks(self):
        rles = []

        for item in self.anns:
            try:
                rles.append(frPyObjects(item['segmentation'], self.origin_height, self.origin_width)[0])
            except Exception:
                pass
        if rles == []:
            raise NoLabelException
        else:
            self.masks = decode(rles).astype(np.float)
        self.masks = resize_blob(self.masks, self.image.shape[:2])
        if self.flipped:
            self.masks = self.masks[:, :, ::-1]
Exemple #2
0
 def fetch_masks(self):
     rles = []
     cls = []
     for item in self.anns:
         try:
             rles.append(
                 frPyObjects(item['segmentation'], self.origin_height,
                             self.origin_width)[0])
             cls.append(self.cats_to_labels[item['category_id']])
         except Exception as e:
             # ignore crowd
             # print 'ignore crowd', item['iscrowd']
             pass
     self.cls = cls
     self.masks = decode(rles).astype(np.float)
     self.masks = resize_blob(self.masks, self.image.shape[:2])
     if self.flipped:
         self.masks = self.masks[:, :, ::-1]
Exemple #3
0
                    'params/' + args.init_weights, caffe.TEST)

    # load config
    if os.path.exists("configs/%s.json" % args.model):
        load_config("configs/%s.json" % args.model)
    else:
        print("Specified config does not exists, use the default config...")

    image = load_image(args.input_image)
    oh, ow = image.shape[:2]
    im_scale = config.TEST_SCALE * 1.0 / max(oh, ow)
    input_blob = image - config.RGB_MEAN
    input_blob = input_blob.transpose((2, 0, 1))
    ih, iw = int(oh * im_scale), int(ow * im_scale)
    ih, iw = ih - ih % 4, iw - iw % 4
    input_blob = resize_blob(input_blob, dest_shape=(ih, iw))
    input_blob = input_blob[np.newaxis, ...]

    ret_masks, ret_scores = gen_masks(net,
                                      input_blob,
                                      config,
                                      dest_shape=(oh, ow))

    # nms
    encoded_masks = encode(ret_masks)
    reserved = np.ones((len(ret_masks)))
    for i in range(len(reserved)):
        if ret_scores[i] < args.threshold:
            reserved[i] = 0
            continue
        if reserved[i]:
Exemple #4
0
def gen_masks_new(net, input, config, dest_shape=None, image=None):
    net.blobs['data'].reshape(*input.shape)
    net.blobs['data'].data[...] = input

    net.forward()

    ih, iw = input.shape[2:]
    if dest_shape != None:
        oh, ow = dest_shape
    else:
        oh, ow = ih, iw
    ih, iw, oh, ow = int(ih), int(iw), int(oh), int(ow)

    if hasattr(config, 'TEST_RFs'):
        ratios = config.TEST_RFs
    else:
        ratios = config.RFs

    ratios = [int(ratio) for ratio in ratios]
    scales = []
    for rf in ratios:
        scales.append(((ih // rf) + (ih % rf > 0), (iw // rf) + (iw % rf > 0)))

    _ = 0
    dynamicK = 1000
    if len(net.blobs['objn'].data) < dynamicK:
        dynamicK = len(net.blobs['objn'].data)
    #determine dynamically how many windows are sampled at test time
    #might be less than 1000
    ret_masks = np.zeros((dynamicK, oh, ow), dtype=np.uint8)
    ret_scores = np.zeros((dynamicK))

    for topk in net.blobs['top_k'].data[:, 0, 0, 0][:dynamicK]:
        if topk < net.blobs['obj_indices'].data[...].shape[0]:
            bid = net.blobs['obj_indices'].data[int(topk), 0, 0, 0]
            bid = int(bid)
            score = float(net.blobs['objn'].data[int(topk)])
            scale_idx = 0
            h, w = scales[scale_idx]
            ceiling = (h + 1) * (w + 1)
            while bid >= ceiling:
                bid -= ceiling
                scale_idx += 1
                try:
                    h, w = scales[scale_idx]
                except Exception as e:
                    raise e
                ceiling = (h + 1) * (w + 1)

            stride = ratios[scale_idx]
            x = bid // (w + 1)
            y = bid % (w + 1)

            xb, xe = (x - config.SLIDING_WINDOW_SIZE // 2) * stride, (
                x + config.SLIDING_WINDOW_SIZE // 2) * stride
            yb, ye = (y - config.SLIDING_WINDOW_SIZE // 2) * stride, (
                y + config.SLIDING_WINDOW_SIZE // 2) * stride
            xb, xe, yb, ye = int(round(1.0 * xb * oh / ih)), int(
                round(1.0 * xe * oh / ih)), int(round(
                    1.0 * yb * ow / iw)), int(round(1.0 * ye * ow / iw))
            size = xe - xb, ye - yb

            mask = net.blobs['masks'].data[_]
            mask = resize_blob(mask, size)
            mask = crop(mask, (xb, xe, yb, ye), (oh, ow))[0]
            xb = max(0, xb)
            xe = min(oh, xe)
            yb = max(0, yb)
            ye = min(ow, ye)
            mask[mask < 0.2] = 0
            mask[mask >= 0.2] = 1
            ret_masks[_, xb:xe, yb:ye] = mask

            ret_scores[_] = score
            _ += 1

    return ret_masks, ret_scores
    def assign_gt(self):
        n, h, w, ratio = len(self.masks), self.feat_h, self.feat_w, self.ratio

        gt_objns = np.ones((h * w))
        gt_objns[np.where(self.objn_match.any(axis=1) == 0)] = 0
        self.gt_objns = gt_objns

        try:
            assert (np.nonzero(self.gt_objns)[0] == np.nonzero(
                self.objn_match.any(axis=1))[0]).all()
        except Exception as e:
            print np.nonzero(self.gt_objns)[0], np.nonzero(
                self.objn_match.any(axis=1))[0]
            raise e

        # mask_filter
        mask_filter = np.ones((h * w))
        mask_filter[np.where(self.mask_match.any(axis=1) == 0)] = 0
        mask_ids = np.where(mask_filter == 1)[0]
        self.mask_filter = mask_filter

        # masks
        mask_scale = 1.0 / ratio * MASK_SIZE / SLIDING_WINDOW_SIZE
        masks = resize_blob(self.masks, None, mask_scale)
        mh, mw = masks.shape[1:]

        # pad
        pad_masks = np.zeros(
            (n, int(mh + MASK_SIZE * 1.5), int(mw + MASK_SIZE * 1.5)), np.int8)
        pad_masks[:, MASK_SIZE / 2:mh + MASK_SIZE / 2,
                  MASK_SIZE / 2:mw + MASK_SIZE / 2] = masks
        masks = pad_masks

        # gt masks
        self.gt_masks = np.zeros((h * w, MASK_SIZE, MASK_SIZE), np.int8)
        obj_ids = np.argmax(self.mask_match[mask_ids, :], axis=1)
        i = 0
        scale = MASK_SIZE / SLIDING_WINDOW_SIZE
        for idx in mask_ids:
            self.gt_masks[idx, :, :] = masks[obj_ids[i], idx / w *
                                             scale:idx / w * scale + MASK_SIZE,
                                             idx % w * scale:idx % w * scale +
                                             MASK_SIZE]
            i += 1

        # gt attention:
        masks = np.zeros((n, h * scale + MASK_SIZE, w * scale + MASK_SIZE))
        bbs = self.bbs.copy()
        bbs *= mask_scale
        bbs[:, :2] = np.floor(bbs[:, :2])
        bbs[:, 2:] = np.ceil(bbs[:, 2:]).astype(np.int)
        for i in range(n):
            masks[i,
                  int(bbs[i, 0] + MASK_SIZE / 2):int(bbs[i, 2] +
                                                     MASK_SIZE / 2),
                  int(bbs[i, 1] + MASK_SIZE / 2):int(bbs[i, 3] +
                                                     MASK_SIZE / 2)] = 1
        masks = resize_blob(masks, None, 1.0 / scale)
        self.gt_atts = np.zeros(
            (h * w, SLIDING_WINDOW_SIZE, SLIDING_WINDOW_SIZE), np.int8)
        _ = 0
        for idx in mask_ids:
            i = obj_ids[_]
            x = idx / w
            y = idx % w
            try:
                self.gt_atts[idx, :, :] = masks[i, x:x + SLIDING_WINDOW_SIZE,
                                                y:y + SLIDING_WINDOW_SIZE]
            except Exception as e:
                raise e
            _ += 1
Exemple #6
0
            print 'stride: ', stride

            x = bid / (w + 1)
            y = bid % (w + 1)

            SLIDING_WINDOW_SIZE = config.SLIDING_WINDOW_SIZE
            xb, xe = int(round((x - SLIDING_WINDOW_SIZE / 2) * stride)), int(
                round((x + SLIDING_WINDOW_SIZE / 2) * stride))
            yb, ye = int(round((y - SLIDING_WINDOW_SIZE / 2) * stride)), int(
                round((y + SLIDING_WINDOW_SIZE / 2) * stride))
            size = xe - xb, ye - yb

            if args.display == 'mask':
                masks = net.blobs['masks'].data[_]
                masks[masks > 0.2] = 1
                masks[masks <= 0.2] = 0
            else:
                masks = net.blobs['atts'].data[_]
            masks = resize_blob(masks, size, method=cv2.INTER_LANCZOS4)
            masks = crop(masks, (xb, xe, yb, ye), (oh, ow))

            RGB_MEAN = config.RGB_MEAN
            img = net.blobs['data'].data[0, :,
                                         max(xb, 0):min(oh, xe),
                                         max(yb, 0):min(ow, ye)].copy()
            img = img.transpose((1, 2, 0))
            img += RGB_MEAN

            visualize_masks(img, masks)
    def handle_fast_mask_segmentation(self, req):

        if self.net == None:
            try:
                self.net = caffe.Net(self.model_path, self.weight_path,
                                     caffe.TEST)
            except:
                rospy.logerr("Error, cannot load fm net to the GPU")
                self.net = None
                self.service_queue = -1
                return FastMaskSegmentationResponse()

        try:
            image = self.br.imgmsg_to_cv2(req.rgb_img, desired_encoding="bgr8")
            image = image.astype(np.float64)

            if self.gpu_id >= 0:
                caffe.set_mode_gpu()
                caffe.set_device(self.gpu_id)
            else:
                caffe.set_mode_cpu()

            oh, ow = image.shape[:2]
            im_scale = config.TEST_SCALE * 1.0 / max(oh, ow)
            input_blob = image - config.RGB_MEAN
            input_blob = input_blob.transpose((2, 0, 1))
            ih, iw = int(oh * im_scale), int(ow * im_scale)
            ih, iw = ih - ih % 4, iw - iw % 4
            input_blob = resize_blob(input_blob, dest_shape=(ih, iw))
            input_blob = input_blob[np.newaxis, ...]

            ret_masks, ret_scores = gen_masks(self.net,
                                              input_blob,
                                              config,
                                              dest_shape=(oh, ow))

            encoded_masks = encode(ret_masks)
            reserved = np.ones((len(ret_masks)))

            for i in range(len(reserved)):
                if ret_scores[i] < self.threshold:
                    reserved[i] = 0
                    continue
                if reserved[i]:
                    for j in range(i + 1, len(reserved)):
                        if reserved[j] and iou(encoded_masks[i],
                                               encoded_masks[j],
                                               [False]) > 0.5:
                            reserved[j] = 0

            temp_image = image.copy()
            fastmask_bbox_arr = FastMaskBB2DArray()

            for _ in range(len(ret_masks)):
                if ret_scores[_] > self.threshold and reserved[_]:

                    mask = ret_masks[_].copy()
                    bbox = toBbox(mask)
                    x, y, w, h = bbox

                    fastmask_bbox = FastMaskBB2D()

                    fastmask_bbox.bbox.x = x
                    fastmask_bbox.bbox.y = y
                    fastmask_bbox.bbox.w = w
                    fastmask_bbox.bbox.h = h

                    fastmask_bbox.score = ret_scores[_]
                    fastmask_bbox_arr.fm_bbox_arr.append(fastmask_bbox)

                    if self.save_and_display:
                        bbox = [int(x) for x in bbox]
                        mask[mask == 1] = 0.3
                        mask[mask == 0] = 1
                        color = COLORS[_ % len(COLORS)]
                        _color = color & 0xff
                        for k in range(3):
                            image[:, :, k] = image[:, :, k] * mask
                        mask[mask == 1] = 0
                        mask[mask > 0] = 0.7
                        for k in range(3):
                            image[:, :, k] += mask * (color & 0xff)
                            color >>= 8
                        cv2.rectangle(temp_image, (bbox[0], bbox[1]),
                                      (bbox[0] + bbox[2], bbox[1] + bbox[3]),
                                      (_color, _color, _color), 1)

            if self.save_and_display:
                image = image.astype(np.uint8)
                temp_image = temp_image.astype(np.uint8)
                cv2.imwrite(fast_mask_root + "/images/mask_result.jpg", image)
                cv2.imwrite(fast_mask_root + "/images/boundingbox_result.jpg",
                            temp_image)

            self.net = None
            return FastMaskSegmentationResponse(
                segmentation_bbox_arr=fastmask_bbox_arr)

        except cv_bridge.CvBridgeError as e:
            rospy.logerr("CvBridge exception %s", e)
            return FastMaskSegmentationResponse()
Exemple #8
0
def gen_masks(net, input, config, dest_shape=None, image=None):
    net.blobs['data'].reshape(*input.shape)
    net.blobs['data'].data[...] = input
    #timer.tic()
    net.forward()
    #print 'time: ', timer.tac()

    ih, iw = input.shape[2:]
    if dest_shape != None:
        oh, ow = dest_shape
    else:
        oh, ow = ih, iw
    ih, iw, oh, ow = int(ih), int(iw), int(oh), int(ow)

    if hasattr(config, 'TEST_RFs'):
        ratios = config.TEST_RFs
    else:
        ratios = config.RFs

    ratios = [int(ratio) for ratio in ratios]
    scales = []
    for rf in ratios:
        scales.append(((ih // rf) + (ih % rf > 0), (iw // rf) + (iw % rf > 0)))

    order = net.blobs['top_k'].data.flatten()

    ret_masks = np.zeros((net.blobs['top_k'].data.shape[0], oh, ow))
    ret_scores = np.zeros((net.blobs['top_k'].data.shape[0]))
    _ = 0
    for bid in net.blobs['top_k'].data[:, 0, 0, 0]:
        bid = int(bid)
        score = float(net.blobs['objn'].data[bid])
        scale_idx = 0
        h, w = scales[scale_idx]
        ceiling = (h + 1) * (w + 1)
        while bid >= ceiling:
            bid -= ceiling
            scale_idx += 1
            try:
                h, w = scales[scale_idx]
            except Exception as e:
                print scale_idx
                raise e
            ceiling = (h + 1) * (w + 1)

        stride = ratios[scale_idx]

        x = bid // (w + 1)
        y = bid % (w + 1)

        xb, xe = (x - config.SLIDING_WINDOW_SIZE // 2) * stride, (
            x + config.SLIDING_WINDOW_SIZE // 2) * stride
        yb, ye = (y - config.SLIDING_WINDOW_SIZE // 2) * stride, (
            y + config.SLIDING_WINDOW_SIZE // 2) * stride
        xb, xe, yb, ye = int(round(1.0 * xb * oh / ih)), int(
            round(1.0 * xe * oh / ih)), int(round(1.0 * yb * ow / iw)), int(
                round(1.0 * ye * ow / iw))
        size = xe - xb, ye - yb

        mask = net.blobs['masks'].data[_]
        mask = resize_blob(mask, size)
        mask = crop(mask, (xb, xe, yb, ye), (oh, ow))[0]
        xb = max(0, xb)
        xe = min(oh, xe)
        yb = max(0, yb)
        ye = min(ow, ye)
        mask[mask < 0.2] = 0
        mask[mask >= 0.2] = 1
        ret_masks[_, xb:xe, yb:ye] = mask

        ret_scores[_] = score
        _ += 1

    return ret_masks, ret_scores
Exemple #9
0
    def assign_gt(self):
        # feature map size
        n, h, w, ratio = len(self.masks), self.feat_h, self.feat_w, self.ratio

        gt_objns = np.ones((h * w))
        gt_objns[np.where(self.objn_match.any(axis=1) == 0)] = 0
        self.gt_objns = gt_objns

        # categories
        if config.USE_CATS:
            gt_cats = np.zeros((h * w))
            labeled_ids = np.where(self.objn_match.any(axis=1) > 0)[0]
            try:
                gt_cats[labeled_ids] = \
                        np.array([self.cls[self.objn_match[i].argmax()] \
                        for i in labeled_ids])
            except Exception as e:
                raise e
            self.gt_cats = gt_cats

        try:
            assert (np.nonzero(self.gt_objns)[0] == np.nonzero(
                self.objn_match.any(axis=1))[0]).all()
        except Exception as e:
            print np.nonzero(self.gt_objns)[0], np.nonzero(
                self.objn_match.any(axis=1))[0]
            raise e

        # mask_filter
        mask_filter = np.ones((h * w))
        mask_filter[np.where(self.mask_match.any(axis=1) == 0)] = 0
        mask_ids = np.where(mask_filter == 1)[0]
        self.mask_filter = mask_filter

        # masks
        mask_scale = 1.0 / ratio * MASK_SIZE / SLIDING_WINDOW_SIZE
        masks = resize_blob(self.masks, None, mask_scale)
        mh, mw = masks.shape[1:]

        # pad
        pad_masks = np.zeros(
            (n, int(mh + MASK_SIZE * 1.5), int(mw + MASK_SIZE * 1.5)))
        pad_masks[:, MASK_SIZE / 2:mh + MASK_SIZE / 2,
                  MASK_SIZE / 2:mw + MASK_SIZE / 2] = masks
        masks = pad_masks

        # gt masks
        self.gt_masks = np.zeros((h * w, MASK_SIZE, MASK_SIZE))
        # mask_ids = np.where(negtive_samples == 1)[0]
        obj_ids = np.argmax(self.mask_match[mask_ids, :], axis=1)
        i = 0
        scale = MASK_SIZE / SLIDING_WINDOW_SIZE
        for idx in mask_ids:
            self.gt_masks[idx, :, :] = masks[obj_ids[i], idx / w *
                                             scale:idx / w * scale + MASK_SIZE,
                                             idx % w * scale:idx % w * scale +
                                             MASK_SIZE]
            mask = masks[obj_ids[i]]
            i += 1
            # visualization
            '''
            img = cv2.resize(self.image.copy(), None, None, fx=mask_scale, fy=mask_scale, interpolation=cv2.INTER_LINEAR)
            img = img[idx/w*scale-MASK_SIZE/2: idx/w*scale+MASK_SIZE/2, idx%w*scale-MASK_SIZE/2: idx%w*scale+MASK_SIZE/2, :] + RGB_MEAN
            try:
                visualize_masks(img, self.gt_masks[idx:idx+1])
            except Exception :
                pass
                print 'label lies on the edge'
            '''

        # gt attention:
        masks = np.zeros((n, h * scale + MASK_SIZE, w * scale + MASK_SIZE))
        bbs = self.bbs.copy()
        bbs *= mask_scale
        bbs[:, :2] = np.floor(bbs[:, :2])
        bbs[:, 2:] = np.ceil(bbs[:, 2:]).astype(np.int)
        for i in range(n):
            # (h, w) order
            masks[i, bbs[i, 0] + MASK_SIZE / 2:bbs[i, 2] + MASK_SIZE / 2,
                  bbs[i, 1] + MASK_SIZE / 2:bbs[i, 3] + MASK_SIZE / 2] = 1
        masks = resize_blob(masks, None, 1.0 / scale)
        self.gt_atts = np.zeros(
            (h * w, SLIDING_WINDOW_SIZE, SLIDING_WINDOW_SIZE))
        _ = 0
        for idx in mask_ids:
            i = obj_ids[_]
            x = idx / w
            y = idx % w
            try:
                self.gt_atts[idx, :, :] = masks[i, x:x + SLIDING_WINDOW_SIZE,
                                                y:y + SLIDING_WINDOW_SIZE]
            except Exception as e:
                raise e
            _ += 1
            # visualize attetion masks
            '''