def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1):
# Need to work on the CPU, where fp16 isn't supported - cast to float to avoid this
mask = mask.float()
box = box.float()
assert len(box) == 5 # xc yc w h angle
padded_mask, scale = expand_masks(mask[None], padding=padding)
mask = padded_mask
# box = expand_boxes(box[None], scale)[0]
box[2:4] *= scale
w = int(np.round(box[2]))
h = int(np.round(box[3]))
w = max(w, 1)
h = max(h, 1)
# Set shape to [batchxCxHxW]
mask = mask.expand((1, 1, -1, -1))
# Resize mask
mask = mask.to(torch.float32)
mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False)
mask = mask[0][0]
if thresh >= 0:
mask = (mask > thresh).to(dtype=torch.float32)
im_mask = np.zeros((im_h, im_w), dtype=np.float32)
im_mask = paste_rotated_roi_in_image(im_mask, mask.cpu().numpy(), box)
return torch.from_numpy(im_mask).to(mask.device)
def resize(self, size):
try:
iter(size)
except TypeError:
assert isinstance(size, (int, float))
size = size, size
width, height = map(int, size)
assert width > 0
assert height > 0
# TODO: Is this needed?
if len(self.masks) == 0:
resized_masks = self.masks.reshape(0, height, width)
resized_size = width, height
return BinaryMaskList(resized_masks, resized_size)
# Height comes first here!
resized_masks = interpolate(
input=self.masks[None].float(),
size=(height, width),
mode="bilinear",
align_corners=False,
)[0].type_as(self.masks)
resized_size = width, height
return BinaryMaskList(resized_masks, resized_size)
def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1):
padded_mask, scale = expand_masks(mask[None], padding=padding)
mask = padded_mask[0, 0]
box = expand_boxes(box[None], scale)[0]
box = box.to(dtype=torch.int32)
TO_REMOVE = 1
w = int(box[2] - box[0] + TO_REMOVE)
h = int(box[3] - box[1] + TO_REMOVE)
w = max(w, 1)
h = max(h, 1)
# Set shape to [batchxCxHxW]
mask = mask.expand((1, 1, -1, -1))
# Resize mask
mask = mask.to(torch.float32)
mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False)
mask = mask[0][0]
if thresh >= 0:
mask = mask > thresh
else:
# for visualization and debugging, we also
# allow it to return an unmodified mask
mask = (mask * 255).to(torch.uint8)
im_mask = torch.zeros((im_h, im_w), dtype=torch.uint8)
x_0 = max(box[0], 0)
x_1 = min(box[2] + 1, im_w)
y_0 = max(box[1], 0)
y_1 = min(box[3] + 1, im_h)
im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]),
(x_0 - box[0]):(x_1 - box[0])]
return im_mask
def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1):
padded_mask, scale = expand_masks(mask[None], padding=padding)
mask = padded_mask[0, 0]
box = expand_boxes(box[None], scale)[0]
box = box.to(dtype=torch.int32)
TO_REMOVE = 1
w = int(box[2] - box[0] + TO_REMOVE)
h = int(box[3] - box[1] + TO_REMOVE)
w = max(w, 1)
h = max(h, 1)
# Set shape to [batchxCxHxW]
mask = mask.expand((1, 1, -1, -1))
# Resize mask
mask = mask.to(torch.float32)
mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False)
mask = mask[0][0]
im_mask = mask.new_zeros((im_h, im_w))
x_0 = max(box[0], 0)
x_1 = min(box[2] + 1, im_w)
y_0 = max(box[1], 0)
y_1 = min(box[3] + 1, im_h)
im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]),
(x_0 - box[0]):(x_1 - box[0])]
new_box = binmask_to_box(im_mask > thresh)
if new_box.sum() == 0:
print(im_mask.max(), im_mask.min())
#print(new_box, box, im_mask.shape)
return im_mask, new_box
def visualize_attentions(self, images, targets, attention_map, iter,
centerness_pack, centerness, pred_targets):
for image, target, coeff, attention, cp, center, pt in zip(
images.tensors, targets, attention_map[0], attention_map[1],
centerness_pack, centerness, pred_targets):
gt_centerness, gt_bbox, anchor_bbox = cp
masks = target.get_field("masks").instances.masks
masks = masks[:masks.shape[0], :, :]
folder = '/media/fs3017/eeum/nuclei/test'
folder = '/home/feng/data/test'
name = str(iter) + '_' + ''.join(
random.choices(string.ascii_uppercase + string.digits, k=6))
image_name = os.path.join(folder, name + '_image.tif')
mask_name = os.path.join(folder, name + '_masks.tif')
attention_name = os.path.join(folder, name + '_attention.tif')
print(image.shape, masks.shape, coeff.shape, attention.shape)
# print(image.device, masks.device, attention[0].device)
mask, _ = masks.max(dim=0)
mask = mask.to(attention.device)
_, ih, iw = image.shape
pad_mask = mask.new(ih, iw).zero_()
h, w = mask.shape
pad_mask[:h, :w].copy_(mask)
resized_masks = interpolate(
input=pad_mask[None, None].float(),
size=(ih // 4, iw // 4),
mode="bilinear",
align_corners=False,
)[0]
# print(resized_masks.shape)
# att = (resized_masks.reshape(1, ih//4*iw//4) @ attention).reshape(1, 1, ih//4, iw//4)
att = (resized_masks.reshape(1, (ih // 4) * (iw // 4))
@ coeff.transpose(0, 1) @ attention).reshape(
1, 1, ih // 4, iw // 4)
att = (att - att.min()) / (att.max() - att.min())
# att[att < 0.5] = 0.0
save_image(att, attention_name, normalize=True)
if pt:
im = image.detach().cpu().numpy()
im = (im - im.min()) / (im.max() - im.min())
im *= 255.
im = im.astype(np.uint8)
im = np.transpose(im, (1, 2, 0))
# print(pt.bbox)
vis_one_training_image_with_pred(im, image_name, folder,
pt.bbox)
else:
save_image(image[None], image_name, normalize=True)
save_image(resized_masks[None], mask_name, normalize=True)
save_image(center[None],
os.path.join(folder, name + '_center.tif'),
normalize=True)
save_image(gt_centerness[None, None],
os.path.join(folder, name + '_center_gt.tif'),
normalize=True)
def resize(self, size=None):
if box is None:
return SemanticMaskList(self.mask)
width, height = map(int, size)
assert width > 0
assert height > 0
# Height comes first here!
resized_mask = interpolate(
input=self.mask.float(),
size=(height, width),
mode="nearest",
align_corners=False,
)[0].type_as(self.mask)
resized_size = width, height
return SemanticMaskList(resized_mask, resized_size)
def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1):
# Need to work on the CPU, where fp16 isn't supported - cast to float to avoid this
mask = mask.float()
box = box.float()
padded_mask, scale = expand_masks(mask[None], padding=padding)
mask = padded_mask[0, 0]
box = expand_boxes(box[None], scale)[0]
box = box.to(dtype=torch.int32)
TO_REMOVE = 1
w = int(box[2] - box[0] + TO_REMOVE)
h = int(box[3] - box[1] + TO_REMOVE)
w = max(w, 1)
h = max(h, 1)
# Set shape to [batchxCxHxW]
mask = mask.expand((1, 1, -1, -1))
# # Resize mask
mask = mask.to(torch.float32)
mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False)
mask = mask[0][0]
if thresh >= 0:
#should be here
mask = mask > thresh
else:
# for visualization and debugging, we also
# allow it to return an unmodified mask
mask = (mask * 255).to(torch.bool)
im_mask = np.zeros((im_h, im_w))
# im_mask = torch.zeros((im_h, im_w), dtype=torch.bool)
x_0 = max(box[0], 0)
x_1 = min(box[2] + 1, im_w)
y_0 = max(box[1], 0)
y_1 = min(box[3] + 1, im_h)
im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]),
(x_0 - box[0]):(x_1 - box[0])]
del mask
return im_mask
def forward(self, masks, boxes, feature_map, padding=1):
assert self.cfg.MODEL.PANOPTIC.PS_ON == True
for mask, box in zip(masks.unsqueeze(1), boxes.bbox):
box = box.numpy().astype(np.int32)
_, _, im_h, im_w = feature_map.size()
TO_REMOVE = 1
w = box[2] - box[0] + TO_REMOVE
h = box[3] - box[1] + TO_REMOVE
w = max(w, 1)
h = max(h, 1)
resized_mask = interpolate(mask, [w, h], mode='bilinear')
x_0 = box[0]
x_1 = min(box[2] + 1, im_w)
y_0 = box[1]
y_1 = min(box[3] + 1, im_h)
mask_cropped = resized_mask[:, :, 0:min(w, im_w - y_0),
0:min(h, im_h - x_0)]
if not self.cfg.MODEL.PANOPTIC.ATTENTION_FUSION:
feature_map[:, :, y_0:min(y_0 + w, im_w),
x_0:min(x_0 + h, im_h)] = mask_cropped
else:
mask_cropped_att = mask_cropped.sigmoid().detach()
feature_map[:, :, y_0:min(y_0 + w, im_w),
x_0:min(x_0 + h, im_h)] *= (1 + mask_cropped_att)
if not self.cfg.MODEL.PANOPTIC.ATTENTION_FUSION:
out_sem_feature_fusion = feature_map
else:
out_sem_feature_fusion = self.sem_block_3x3(F.relu(feature_map))
out_sem_feature_fusion = self.sem_block_1x1(
F.relu(out_sem_feature_fusion))
return out_sem_feature_fusion
def __init__(self, masks, size):
"""
Arguments:
masks: Either torch.tensor of [num_instances, H, W]
or list of torch.tensors of [H, W] with num_instances elems,
or RLE (Run Length Encoding) - interpreted as list of dicts,
or BinaryMaskList.
size: absolute image size, width first
After initialization, a hard copy will be made, to leave the
initializing source data intact.
"""
assert isinstance(size, (list, tuple))
assert len(size) == 2
if isinstance(masks, torch.Tensor):
# The raw data representation is passed as argument
masks = masks.clone()
elif isinstance(masks, (list, tuple)):
if len(masks) == 0:
masks = torch.empty([0, size[1],
size[0]]) # num_instances = 0!
elif isinstance(masks[0], torch.Tensor):
masks = torch.stack(masks, dim=2).clone()
elif isinstance(masks[0], dict) and "counts" in masks[0]:
# RLE interpretation
rle_sizes = [tuple(inst["size"]) for inst in masks]
masks = mask_utils.decode(masks) # [h, w, n]
masks = torch.tensor(masks).permute(2, 0, 1) # [n, h, w]
assert rle_sizes.count(rle_sizes[0]) == len(rle_sizes), (
"All the sizes must be the same size: %s" % rle_sizes)
# in RLE, height come first in "size"
rle_height, rle_width = rle_sizes[0]
assert masks.shape[1] == rle_height
assert masks.shape[2] == rle_width
width, height = size
if width != rle_width or height != rle_height:
masks = interpolate(
input=masks[None].float(),
size=(height, width),
mode="bilinear",
align_corners=False,
)[0].type_as(masks)
else:
RuntimeError(
"Type of `masks[0]` could not be interpreted: %s" %
type(masks))
elif isinstance(masks, BinaryMaskList):
# just hard copy the BinaryMaskList instance's underlying data
masks = masks.masks.clone()
else:
RuntimeError(
"Type of `masks` argument could not be interpreted:%s" %
type(masks))
if len(masks.shape) == 2:
# if only a single instance mask is passed
masks = masks[None]
assert len(masks.shape) == 3
assert masks.shape[1] == size[1], "%s != %s" % (masks.shape[1],
size[1])
assert masks.shape[2] == size[0], "%s != %s" % (masks.shape[2],
size[0])
self.masks = masks
self.size = tuple(size)
def paste_mask_in_image(mask,
box,
im_h,
im_w,
thresh=0.5,
padding=1,
do_CRF=False,
crf_postprocessor=None,
img_single=None,
score=None,
upscale=1.0,
img_id=0):
# Need to work on the CPU, where fp16 isn't supported - cast to float to avoid this
# print("box score", score)
mask = mask.float()
CRF_normalize = True
CRF_zoom = 3.0
CRF_zoom_th = 20000
dataset = 'ss'
if dataset == 'city':
print("CITYSCAPES!!!")
# if CRF_normalize:
# mask = (mask - mask.min()) / mask.max()
#
# if upscale != 1.0:
# mask = torch.clamp(mask*upscale+0.5, min=0, max=1)
box = box.float()
padded_mask, scale = expand_masks(mask[None], padding=padding)
mask = padded_mask[0, 0]
box = expand_boxes(box[None], scale)[0]
box = box.to(dtype=torch.int32)
box[0] = max(box[0], 0)
box[1] = max(box[1], 0)
box[2] = min(box[2], im_w)
box[3] = min(box[3], im_h)
# print(im_w, im_h)
TO_REMOVE = 1
w = int(box[2] - box[0] + TO_REMOVE)
h = int(box[3] - box[1] + TO_REMOVE)
w = max(w, 1)
h = max(h, 1)
# Set shape to [batchxCxHxW]
mask = mask.expand((1, 1, -1, -1))
# Resize mask
mask = mask.to(torch.float32)
mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False)
mask = mask[0][0]
# print(torch.max(mask))
if upscale != 1.0:
box_size = w * h
if box_size > 100:
upscale = 0.15
# elif box_size > 40*40:
# upscale = 0.2
else:
upscale = 0.2
# print(upscale)
threshold = sorted(mask.flatten(), reverse=True)[int(
box_size * upscale)] + 1e-8
mask = mask / threshold
mask = torch.clamp(mask, 0, 1)
# CRF here!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
if do_CRF and score > 0.0:
img_single = img_single[:, :, ::-1]
h_orig, w_orig = img_single.shape[:2]
im_mask = torch.zeros((im_h, im_w))
x_0 = max(box[0], 0)
x_1 = min(box[2] + 1, im_w)
y_0 = max(box[1], 0)
y_1 = min(box[3] + 1, im_h)
# print(x_1, y_1, box[2], box[3])
p_box = [int(x_0), int(y_0), int(x_1), int(y_1)]
im_mask[y_0:y_1,
x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]),
(x_0 - box[0]):(x_1 - box[0])] # projection
im_mask = im_mask.numpy()
if dataset == 'city':
img_crop, cut_point_crop, img_resize, p_box_crop, mask_crop, resize_ratio = crop_city_roi(
p_box, img_single, mask=im_mask, return_ratio=True)
# img_crop_norm = norm_image(img_crop)
h_new, w_new = img_resize.shape[:2]
mask_for_crf = np.concatenate([
np.expand_dims(mask_crop, 0),
np.expand_dims(1 - mask_crop, 0)
],
axis=0)
mask_for_crf = crf_postprocessor(img_crop, mask_for_crf)[0]
# mask_for_crf_norm = crf_postprocessor(img_crop_norm, mask_for_crf)[0]
# cv2.imshow('img_crop', cv2.resize(img_crop, dsize=None, fx=0.5, fy=0.5))
# cv2.imshow('img_crop_norm', cv2.resize(img_crop_norm, dsize=None, fx=0.5, fy=0.5))
#
# cv2.imshow('mask_crop', cv2.resize((mask_crop * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5))
# cv2.imshow('crf_crop', cv2.resize((mask_for_crf_nonorm * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5))
# cv2.imshow('crf_crop_norm', cv2.resize((mask_for_crf_norm * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5))
# mask_for_crf = mask_for_crf_nonorm
total_mask = np.zeros([h_new, w_new])
total_mask[cut_point_crop[1]:cut_point_crop[3],
cut_point_crop[0]:cut_point_crop[2]] = mask_for_crf
mask_for_crf = cv2.resize(total_mask, (2048, 1024))
# cv2.imshow('img', cv2.resize(img_single, dsize=None, fx=0.5, fy=0.5))
# cv2.imshow('mask', cv2.resize((im_mask * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5))
# cv2.imshow('crf', cv2.resize((mask_for_crf * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5))
# cv2.waitKey(0)
else:
if CRF_zoom != 1.0 and box_size < CRF_zoom_th:
cut_point = [
int(box[0] * (1 - 1 / CRF_zoom)),
int(box[1] * (1 - 1 / CRF_zoom)),
int((w_orig + (CRF_zoom - 1) * box[2]) / CRF_zoom),
int((h_orig + (CRF_zoom - 1) * box[3]) / CRF_zoom)
]
before_zoom_size = (cut_point[2] - cut_point[0],
cut_point[3] - cut_point[1])
image_zoom = img_single[cut_point[1]:cut_point[3],
cut_point[0]:cut_point[2]]
mask_zoom = im_mask[cut_point[1]:cut_point[3],
cut_point[0]:cut_point[2]]
image_zoom = cv2.resize(image_zoom, (w_orig, h_orig))
mask_zoom = cv2.resize(mask_zoom, (w_orig, h_orig))
mask_for_crf = np.concatenate([
np.expand_dims(mask_zoom, 0),
np.expand_dims(1 - mask_zoom, 0)
],
axis=0)
mask_for_crf = crf_postprocessor(image_zoom, mask_for_crf)[0]
mask_zoom = cv2.resize(mask_for_crf.copy(), before_zoom_size)
mask_for_crf = np.zeros(im_mask.shape)
# print(w_orig, h_orig)
# print(mask_zoom.shape)
mask_for_crf[cut_point[1]:cut_point[3],
cut_point[0]:cut_point[2]] = mask_zoom
save_ratio = 1.0
else:
mask_for_crf = np.concatenate([
np.expand_dims(im_mask, 0),
np.expand_dims(1 - im_mask, 0)
],
axis=0)
mask_for_crf = crf_postprocessor(img_single, mask_for_crf)[0]
# cv2.imshow('img', img_single)
# # cv2.imshow('mask', (im_mask * 255.0).astype('uint8'))
# cv2.imshow('mask_crf_nozoom', (mask_for_crf * 255.0).astype('uint8'))
#
# cv2.waitKey(0)
mask = torch.Tensor(mask_for_crf)
if thresh >= 0:
im_mask = mask > thresh
else:
# for visualization and debugging, we also
# allow it to return an unmodified mask
im_mask = (mask * 255).to(torch.bool)
else:
if thresh >= 0:
mask = mask > thresh
else:
# for visualization and debugging, we also
# allow it to return an unmodified mask
mask = (mask * 255).to(torch.bool)
im_mask = torch.zeros((im_h, im_w), dtype=torch.bool)
x_0 = max(box[0], 0)
x_1 = min(box[2] + 1, im_w)
y_0 = max(box[1], 0)
y_1 = min(box[3] + 1, im_h)
im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]),
(x_0 - box[0]):(x_1 - box[0])]
return im_mask
