def fetch_masks(self):
rles = []
for item in self.anns:
try:
rles.append(frPyObjects(item['segmentation'], self.origin_height, self.origin_width)[0])
except Exception:
pass
if rles == []:
raise NoLabelException
else:
self.masks = decode(rles).astype(np.float)
self.masks = resize_blob(self.masks, self.image.shape[:2])
if self.flipped:
self.masks = self.masks[:, :, ::-1]
def fetch_masks(self):
rles = []
cls = []
for item in self.anns:
try:
rles.append(
frPyObjects(item['segmentation'], self.origin_height,
self.origin_width)[0])
cls.append(self.cats_to_labels[item['category_id']])
except Exception as e:
# ignore crowd
# print 'ignore crowd', item['iscrowd']
pass
self.cls = cls
self.masks = decode(rles).astype(np.float)
self.masks = resize_blob(self.masks, self.image.shape[:2])
if self.flipped:
self.masks = self.masks[:, :, ::-1]
'params/' + args.init_weights, caffe.TEST)
# load config
if os.path.exists("configs/%s.json" % args.model):
load_config("configs/%s.json" % args.model)
else:
print("Specified config does not exists, use the default config...")
image = load_image(args.input_image)
oh, ow = image.shape[:2]
im_scale = config.TEST_SCALE * 1.0 / max(oh, ow)
input_blob = image - config.RGB_MEAN
input_blob = input_blob.transpose((2, 0, 1))
ih, iw = int(oh * im_scale), int(ow * im_scale)
ih, iw = ih - ih % 4, iw - iw % 4
input_blob = resize_blob(input_blob, dest_shape=(ih, iw))
input_blob = input_blob[np.newaxis, ...]
ret_masks, ret_scores = gen_masks(net,
input_blob,
config,
dest_shape=(oh, ow))
# nms
encoded_masks = encode(ret_masks)
reserved = np.ones((len(ret_masks)))
for i in range(len(reserved)):
if ret_scores[i] < args.threshold:
reserved[i] = 0
continue
if reserved[i]:
def gen_masks_new(net, input, config, dest_shape=None, image=None):
net.blobs['data'].reshape(*input.shape)
net.blobs['data'].data[...] = input
net.forward()
ih, iw = input.shape[2:]
if dest_shape != None:
oh, ow = dest_shape
else:
oh, ow = ih, iw
ih, iw, oh, ow = int(ih), int(iw), int(oh), int(ow)
if hasattr(config, 'TEST_RFs'):
ratios = config.TEST_RFs
else:
ratios = config.RFs
ratios = [int(ratio) for ratio in ratios]
scales = []
for rf in ratios:
scales.append(((ih // rf) + (ih % rf > 0), (iw // rf) + (iw % rf > 0)))
_ = 0
dynamicK = 1000
if len(net.blobs['objn'].data) < dynamicK:
dynamicK = len(net.blobs['objn'].data)
#determine dynamically how many windows are sampled at test time
#might be less than 1000
ret_masks = np.zeros((dynamicK, oh, ow), dtype=np.uint8)
ret_scores = np.zeros((dynamicK))
for topk in net.blobs['top_k'].data[:, 0, 0, 0][:dynamicK]:
if topk < net.blobs['obj_indices'].data[...].shape[0]:
bid = net.blobs['obj_indices'].data[int(topk), 0, 0, 0]
bid = int(bid)
score = float(net.blobs['objn'].data[int(topk)])
scale_idx = 0
h, w = scales[scale_idx]
ceiling = (h + 1) * (w + 1)
while bid >= ceiling:
bid -= ceiling
scale_idx += 1
try:
h, w = scales[scale_idx]
except Exception as e:
raise e
ceiling = (h + 1) * (w + 1)
stride = ratios[scale_idx]
x = bid // (w + 1)
y = bid % (w + 1)
xb, xe = (x - config.SLIDING_WINDOW_SIZE // 2) * stride, (
x + config.SLIDING_WINDOW_SIZE // 2) * stride
yb, ye = (y - config.SLIDING_WINDOW_SIZE // 2) * stride, (
y + config.SLIDING_WINDOW_SIZE // 2) * stride
xb, xe, yb, ye = int(round(1.0 * xb * oh / ih)), int(
round(1.0 * xe * oh / ih)), int(round(
1.0 * yb * ow / iw)), int(round(1.0 * ye * ow / iw))
size = xe - xb, ye - yb
mask = net.blobs['masks'].data[_]
mask = resize_blob(mask, size)
mask = crop(mask, (xb, xe, yb, ye), (oh, ow))[0]
xb = max(0, xb)
xe = min(oh, xe)
yb = max(0, yb)
ye = min(ow, ye)
mask[mask < 0.2] = 0
mask[mask >= 0.2] = 1
ret_masks[_, xb:xe, yb:ye] = mask
ret_scores[_] = score
_ += 1
return ret_masks, ret_scores
def assign_gt(self):
n, h, w, ratio = len(self.masks), self.feat_h, self.feat_w, self.ratio
gt_objns = np.ones((h * w))
gt_objns[np.where(self.objn_match.any(axis=1) == 0)] = 0
self.gt_objns = gt_objns
try:
assert (np.nonzero(self.gt_objns)[0] == np.nonzero(
self.objn_match.any(axis=1))[0]).all()
except Exception as e:
print np.nonzero(self.gt_objns)[0], np.nonzero(
self.objn_match.any(axis=1))[0]
raise e
# mask_filter
mask_filter = np.ones((h * w))
mask_filter[np.where(self.mask_match.any(axis=1) == 0)] = 0
mask_ids = np.where(mask_filter == 1)[0]
self.mask_filter = mask_filter
# masks
mask_scale = 1.0 / ratio * MASK_SIZE / SLIDING_WINDOW_SIZE
masks = resize_blob(self.masks, None, mask_scale)
mh, mw = masks.shape[1:]
# pad
pad_masks = np.zeros(
(n, int(mh + MASK_SIZE * 1.5), int(mw + MASK_SIZE * 1.5)), np.int8)
pad_masks[:, MASK_SIZE / 2:mh + MASK_SIZE / 2,
MASK_SIZE / 2:mw + MASK_SIZE / 2] = masks
masks = pad_masks
# gt masks
self.gt_masks = np.zeros((h * w, MASK_SIZE, MASK_SIZE), np.int8)
obj_ids = np.argmax(self.mask_match[mask_ids, :], axis=1)
i = 0
scale = MASK_SIZE / SLIDING_WINDOW_SIZE
for idx in mask_ids:
self.gt_masks[idx, :, :] = masks[obj_ids[i], idx / w *
scale:idx / w * scale + MASK_SIZE,
idx % w * scale:idx % w * scale +
MASK_SIZE]
i += 1
# gt attention:
masks = np.zeros((n, h * scale + MASK_SIZE, w * scale + MASK_SIZE))
bbs = self.bbs.copy()
bbs *= mask_scale
bbs[:, :2] = np.floor(bbs[:, :2])
bbs[:, 2:] = np.ceil(bbs[:, 2:]).astype(np.int)
for i in range(n):
masks[i,
int(bbs[i, 0] + MASK_SIZE / 2):int(bbs[i, 2] +
MASK_SIZE / 2),
int(bbs[i, 1] + MASK_SIZE / 2):int(bbs[i, 3] +
MASK_SIZE / 2)] = 1
masks = resize_blob(masks, None, 1.0 / scale)
self.gt_atts = np.zeros(
(h * w, SLIDING_WINDOW_SIZE, SLIDING_WINDOW_SIZE), np.int8)
_ = 0
for idx in mask_ids:
i = obj_ids[_]
x = idx / w
y = idx % w
try:
self.gt_atts[idx, :, :] = masks[i, x:x + SLIDING_WINDOW_SIZE,
y:y + SLIDING_WINDOW_SIZE]
except Exception as e:
raise e
_ += 1
print 'stride: ', stride
x = bid / (w + 1)
y = bid % (w + 1)
SLIDING_WINDOW_SIZE = config.SLIDING_WINDOW_SIZE
xb, xe = int(round((x - SLIDING_WINDOW_SIZE / 2) * stride)), int(
round((x + SLIDING_WINDOW_SIZE / 2) * stride))
yb, ye = int(round((y - SLIDING_WINDOW_SIZE / 2) * stride)), int(
round((y + SLIDING_WINDOW_SIZE / 2) * stride))
size = xe - xb, ye - yb
if args.display == 'mask':
masks = net.blobs['masks'].data[_]
masks[masks > 0.2] = 1
masks[masks <= 0.2] = 0
else:
masks = net.blobs['atts'].data[_]
masks = resize_blob(masks, size, method=cv2.INTER_LANCZOS4)
masks = crop(masks, (xb, xe, yb, ye), (oh, ow))
RGB_MEAN = config.RGB_MEAN
img = net.blobs['data'].data[0, :,
max(xb, 0):min(oh, xe),
max(yb, 0):min(ow, ye)].copy()
img = img.transpose((1, 2, 0))
img += RGB_MEAN
visualize_masks(img, masks)
def handle_fast_mask_segmentation(self, req):
if self.net == None:
try:
self.net = caffe.Net(self.model_path, self.weight_path,
caffe.TEST)
except:
rospy.logerr("Error, cannot load fm net to the GPU")
self.net = None
self.service_queue = -1
return FastMaskSegmentationResponse()
try:
image = self.br.imgmsg_to_cv2(req.rgb_img, desired_encoding="bgr8")
image = image.astype(np.float64)
if self.gpu_id >= 0:
caffe.set_mode_gpu()
caffe.set_device(self.gpu_id)
else:
caffe.set_mode_cpu()
oh, ow = image.shape[:2]
im_scale = config.TEST_SCALE * 1.0 / max(oh, ow)
input_blob = image - config.RGB_MEAN
input_blob = input_blob.transpose((2, 0, 1))
ih, iw = int(oh * im_scale), int(ow * im_scale)
ih, iw = ih - ih % 4, iw - iw % 4
input_blob = resize_blob(input_blob, dest_shape=(ih, iw))
input_blob = input_blob[np.newaxis, ...]
ret_masks, ret_scores = gen_masks(self.net,
input_blob,
config,
dest_shape=(oh, ow))
encoded_masks = encode(ret_masks)
reserved = np.ones((len(ret_masks)))
for i in range(len(reserved)):
if ret_scores[i] < self.threshold:
reserved[i] = 0
continue
if reserved[i]:
for j in range(i + 1, len(reserved)):
if reserved[j] and iou(encoded_masks[i],
encoded_masks[j],
[False]) > 0.5:
reserved[j] = 0
temp_image = image.copy()
fastmask_bbox_arr = FastMaskBB2DArray()
for _ in range(len(ret_masks)):
if ret_scores[_] > self.threshold and reserved[_]:
mask = ret_masks[_].copy()
bbox = toBbox(mask)
x, y, w, h = bbox
fastmask_bbox = FastMaskBB2D()
fastmask_bbox.bbox.x = x
fastmask_bbox.bbox.y = y
fastmask_bbox.bbox.w = w
fastmask_bbox.bbox.h = h
fastmask_bbox.score = ret_scores[_]
fastmask_bbox_arr.fm_bbox_arr.append(fastmask_bbox)
if self.save_and_display:
bbox = [int(x) for x in bbox]
mask[mask == 1] = 0.3
mask[mask == 0] = 1
color = COLORS[_ % len(COLORS)]
_color = color & 0xff
for k in range(3):
image[:, :, k] = image[:, :, k] * mask
mask[mask == 1] = 0
mask[mask > 0] = 0.7
for k in range(3):
image[:, :, k] += mask * (color & 0xff)
color >>= 8
cv2.rectangle(temp_image, (bbox[0], bbox[1]),
(bbox[0] + bbox[2], bbox[1] + bbox[3]),
(_color, _color, _color), 1)
if self.save_and_display:
image = image.astype(np.uint8)
temp_image = temp_image.astype(np.uint8)
cv2.imwrite(fast_mask_root + "/images/mask_result.jpg", image)
cv2.imwrite(fast_mask_root + "/images/boundingbox_result.jpg",
temp_image)
self.net = None
return FastMaskSegmentationResponse(
segmentation_bbox_arr=fastmask_bbox_arr)
except cv_bridge.CvBridgeError as e:
rospy.logerr("CvBridge exception %s", e)
return FastMaskSegmentationResponse()
def gen_masks(net, input, config, dest_shape=None, image=None):
net.blobs['data'].reshape(*input.shape)
net.blobs['data'].data[...] = input
#timer.tic()
net.forward()
#print 'time: ', timer.tac()
ih, iw = input.shape[2:]
if dest_shape != None:
oh, ow = dest_shape
else:
oh, ow = ih, iw
ih, iw, oh, ow = int(ih), int(iw), int(oh), int(ow)
if hasattr(config, 'TEST_RFs'):
ratios = config.TEST_RFs
else:
ratios = config.RFs
ratios = [int(ratio) for ratio in ratios]
scales = []
for rf in ratios:
scales.append(((ih // rf) + (ih % rf > 0), (iw // rf) + (iw % rf > 0)))
order = net.blobs['top_k'].data.flatten()
ret_masks = np.zeros((net.blobs['top_k'].data.shape[0], oh, ow))
ret_scores = np.zeros((net.blobs['top_k'].data.shape[0]))
_ = 0
for bid in net.blobs['top_k'].data[:, 0, 0, 0]:
bid = int(bid)
score = float(net.blobs['objn'].data[bid])
scale_idx = 0
h, w = scales[scale_idx]
ceiling = (h + 1) * (w + 1)
while bid >= ceiling:
bid -= ceiling
scale_idx += 1
try:
h, w = scales[scale_idx]
except Exception as e:
print scale_idx
raise e
ceiling = (h + 1) * (w + 1)
stride = ratios[scale_idx]
x = bid // (w + 1)
y = bid % (w + 1)
xb, xe = (x - config.SLIDING_WINDOW_SIZE // 2) * stride, (
x + config.SLIDING_WINDOW_SIZE // 2) * stride
yb, ye = (y - config.SLIDING_WINDOW_SIZE // 2) * stride, (
y + config.SLIDING_WINDOW_SIZE // 2) * stride
xb, xe, yb, ye = int(round(1.0 * xb * oh / ih)), int(
round(1.0 * xe * oh / ih)), int(round(1.0 * yb * ow / iw)), int(
round(1.0 * ye * ow / iw))
size = xe - xb, ye - yb
mask = net.blobs['masks'].data[_]
mask = resize_blob(mask, size)
mask = crop(mask, (xb, xe, yb, ye), (oh, ow))[0]
xb = max(0, xb)
xe = min(oh, xe)
yb = max(0, yb)
ye = min(ow, ye)
mask[mask < 0.2] = 0
mask[mask >= 0.2] = 1
ret_masks[_, xb:xe, yb:ye] = mask
ret_scores[_] = score
_ += 1
return ret_masks, ret_scores
def assign_gt(self):
# feature map size
n, h, w, ratio = len(self.masks), self.feat_h, self.feat_w, self.ratio
gt_objns = np.ones((h * w))
gt_objns[np.where(self.objn_match.any(axis=1) == 0)] = 0
self.gt_objns = gt_objns
# categories
if config.USE_CATS:
gt_cats = np.zeros((h * w))
labeled_ids = np.where(self.objn_match.any(axis=1) > 0)[0]
try:
gt_cats[labeled_ids] = \
np.array([self.cls[self.objn_match[i].argmax()] \
for i in labeled_ids])
except Exception as e:
raise e
self.gt_cats = gt_cats
try:
assert (np.nonzero(self.gt_objns)[0] == np.nonzero(
self.objn_match.any(axis=1))[0]).all()
except Exception as e:
print np.nonzero(self.gt_objns)[0], np.nonzero(
self.objn_match.any(axis=1))[0]
raise e
# mask_filter
mask_filter = np.ones((h * w))
mask_filter[np.where(self.mask_match.any(axis=1) == 0)] = 0
mask_ids = np.where(mask_filter == 1)[0]
self.mask_filter = mask_filter
# masks
mask_scale = 1.0 / ratio * MASK_SIZE / SLIDING_WINDOW_SIZE
masks = resize_blob(self.masks, None, mask_scale)
mh, mw = masks.shape[1:]
# pad
pad_masks = np.zeros(
(n, int(mh + MASK_SIZE * 1.5), int(mw + MASK_SIZE * 1.5)))
pad_masks[:, MASK_SIZE / 2:mh + MASK_SIZE / 2,
MASK_SIZE / 2:mw + MASK_SIZE / 2] = masks
masks = pad_masks
# gt masks
self.gt_masks = np.zeros((h * w, MASK_SIZE, MASK_SIZE))
# mask_ids = np.where(negtive_samples == 1)[0]
obj_ids = np.argmax(self.mask_match[mask_ids, :], axis=1)
i = 0
scale = MASK_SIZE / SLIDING_WINDOW_SIZE
for idx in mask_ids:
self.gt_masks[idx, :, :] = masks[obj_ids[i], idx / w *
scale:idx / w * scale + MASK_SIZE,
idx % w * scale:idx % w * scale +
MASK_SIZE]
mask = masks[obj_ids[i]]
i += 1
# visualization
'''
img = cv2.resize(self.image.copy(), None, None, fx=mask_scale, fy=mask_scale, interpolation=cv2.INTER_LINEAR)
img = img[idx/w*scale-MASK_SIZE/2: idx/w*scale+MASK_SIZE/2, idx%w*scale-MASK_SIZE/2: idx%w*scale+MASK_SIZE/2, :] + RGB_MEAN
try:
visualize_masks(img, self.gt_masks[idx:idx+1])
except Exception :
pass
print 'label lies on the edge'
'''
# gt attention:
masks = np.zeros((n, h * scale + MASK_SIZE, w * scale + MASK_SIZE))
bbs = self.bbs.copy()
bbs *= mask_scale
bbs[:, :2] = np.floor(bbs[:, :2])
bbs[:, 2:] = np.ceil(bbs[:, 2:]).astype(np.int)
for i in range(n):
# (h, w) order
masks[i, bbs[i, 0] + MASK_SIZE / 2:bbs[i, 2] + MASK_SIZE / 2,
bbs[i, 1] + MASK_SIZE / 2:bbs[i, 3] + MASK_SIZE / 2] = 1
masks = resize_blob(masks, None, 1.0 / scale)
self.gt_atts = np.zeros(
(h * w, SLIDING_WINDOW_SIZE, SLIDING_WINDOW_SIZE))
_ = 0
for idx in mask_ids:
i = obj_ids[_]
x = idx / w
y = idx % w
try:
self.gt_atts[idx, :, :] = masks[i, x:x + SLIDING_WINDOW_SIZE,
y:y + SLIDING_WINDOW_SIZE]
except Exception as e:
raise e
_ += 1
# visualize attetion masks
'''