def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1): # Need to work on the CPU, where fp16 isn't supported - cast to float to avoid this mask = mask.float() box = box.float() assert len(box) == 5 # xc yc w h angle padded_mask, scale = expand_masks(mask[None], padding=padding) mask = padded_mask # box = expand_boxes(box[None], scale)[0] box[2:4] *= scale w = int(np.round(box[2])) h = int(np.round(box[3])) w = max(w, 1) h = max(h, 1) # Set shape to [batchxCxHxW] mask = mask.expand((1, 1, -1, -1)) # Resize mask mask = mask.to(torch.float32) mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False) mask = mask[0][0] if thresh >= 0: mask = (mask > thresh).to(dtype=torch.float32) im_mask = np.zeros((im_h, im_w), dtype=np.float32) im_mask = paste_rotated_roi_in_image(im_mask, mask.cpu().numpy(), box) return torch.from_numpy(im_mask).to(mask.device)
def resize(self, size): try: iter(size) except TypeError: assert isinstance(size, (int, float)) size = size, size width, height = map(int, size) assert width > 0 assert height > 0 # TODO: Is this needed? if len(self.masks) == 0: resized_masks = self.masks.reshape(0, height, width) resized_size = width, height return BinaryMaskList(resized_masks, resized_size) # Height comes first here! resized_masks = interpolate( input=self.masks[None].float(), size=(height, width), mode="bilinear", align_corners=False, )[0].type_as(self.masks) resized_size = width, height return BinaryMaskList(resized_masks, resized_size)
def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1): padded_mask, scale = expand_masks(mask[None], padding=padding) mask = padded_mask[0, 0] box = expand_boxes(box[None], scale)[0] box = box.to(dtype=torch.int32) TO_REMOVE = 1 w = int(box[2] - box[0] + TO_REMOVE) h = int(box[3] - box[1] + TO_REMOVE) w = max(w, 1) h = max(h, 1) # Set shape to [batchxCxHxW] mask = mask.expand((1, 1, -1, -1)) # Resize mask mask = mask.to(torch.float32) mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False) mask = mask[0][0] if thresh >= 0: mask = mask > thresh else: # for visualization and debugging, we also # allow it to return an unmodified mask mask = (mask * 255).to(torch.uint8) im_mask = torch.zeros((im_h, im_w), dtype=torch.uint8) x_0 = max(box[0], 0) x_1 = min(box[2] + 1, im_w) y_0 = max(box[1], 0) y_1 = min(box[3] + 1, im_h) im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]), (x_0 - box[0]):(x_1 - box[0])] return im_mask
def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1): padded_mask, scale = expand_masks(mask[None], padding=padding) mask = padded_mask[0, 0] box = expand_boxes(box[None], scale)[0] box = box.to(dtype=torch.int32) TO_REMOVE = 1 w = int(box[2] - box[0] + TO_REMOVE) h = int(box[3] - box[1] + TO_REMOVE) w = max(w, 1) h = max(h, 1) # Set shape to [batchxCxHxW] mask = mask.expand((1, 1, -1, -1)) # Resize mask mask = mask.to(torch.float32) mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False) mask = mask[0][0] im_mask = mask.new_zeros((im_h, im_w)) x_0 = max(box[0], 0) x_1 = min(box[2] + 1, im_w) y_0 = max(box[1], 0) y_1 = min(box[3] + 1, im_h) im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]), (x_0 - box[0]):(x_1 - box[0])] new_box = binmask_to_box(im_mask > thresh) if new_box.sum() == 0: print(im_mask.max(), im_mask.min()) #print(new_box, box, im_mask.shape) return im_mask, new_box
def visualize_attentions(self, images, targets, attention_map, iter, centerness_pack, centerness, pred_targets): for image, target, coeff, attention, cp, center, pt in zip( images.tensors, targets, attention_map[0], attention_map[1], centerness_pack, centerness, pred_targets): gt_centerness, gt_bbox, anchor_bbox = cp masks = target.get_field("masks").instances.masks masks = masks[:masks.shape[0], :, :] folder = '/media/fs3017/eeum/nuclei/test' folder = '/home/feng/data/test' name = str(iter) + '_' + ''.join( random.choices(string.ascii_uppercase + string.digits, k=6)) image_name = os.path.join(folder, name + '_image.tif') mask_name = os.path.join(folder, name + '_masks.tif') attention_name = os.path.join(folder, name + '_attention.tif') print(image.shape, masks.shape, coeff.shape, attention.shape) # print(image.device, masks.device, attention[0].device) mask, _ = masks.max(dim=0) mask = mask.to(attention.device) _, ih, iw = image.shape pad_mask = mask.new(ih, iw).zero_() h, w = mask.shape pad_mask[:h, :w].copy_(mask) resized_masks = interpolate( input=pad_mask[None, None].float(), size=(ih // 4, iw // 4), mode="bilinear", align_corners=False, )[0] # print(resized_masks.shape) # att = (resized_masks.reshape(1, ih//4*iw//4) @ attention).reshape(1, 1, ih//4, iw//4) att = (resized_masks.reshape(1, (ih // 4) * (iw // 4)) @ coeff.transpose(0, 1) @ attention).reshape( 1, 1, ih // 4, iw // 4) att = (att - att.min()) / (att.max() - att.min()) # att[att < 0.5] = 0.0 save_image(att, attention_name, normalize=True) if pt: im = image.detach().cpu().numpy() im = (im - im.min()) / (im.max() - im.min()) im *= 255. im = im.astype(np.uint8) im = np.transpose(im, (1, 2, 0)) # print(pt.bbox) vis_one_training_image_with_pred(im, image_name, folder, pt.bbox) else: save_image(image[None], image_name, normalize=True) save_image(resized_masks[None], mask_name, normalize=True) save_image(center[None], os.path.join(folder, name + '_center.tif'), normalize=True) save_image(gt_centerness[None, None], os.path.join(folder, name + '_center_gt.tif'), normalize=True)
def resize(self, size=None): if box is None: return SemanticMaskList(self.mask) width, height = map(int, size) assert width > 0 assert height > 0 # Height comes first here! resized_mask = interpolate( input=self.mask.float(), size=(height, width), mode="nearest", align_corners=False, )[0].type_as(self.mask) resized_size = width, height return SemanticMaskList(resized_mask, resized_size)
def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1): # Need to work on the CPU, where fp16 isn't supported - cast to float to avoid this mask = mask.float() box = box.float() padded_mask, scale = expand_masks(mask[None], padding=padding) mask = padded_mask[0, 0] box = expand_boxes(box[None], scale)[0] box = box.to(dtype=torch.int32) TO_REMOVE = 1 w = int(box[2] - box[0] + TO_REMOVE) h = int(box[3] - box[1] + TO_REMOVE) w = max(w, 1) h = max(h, 1) # Set shape to [batchxCxHxW] mask = mask.expand((1, 1, -1, -1)) # # Resize mask mask = mask.to(torch.float32) mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False) mask = mask[0][0] if thresh >= 0: #should be here mask = mask > thresh else: # for visualization and debugging, we also # allow it to return an unmodified mask mask = (mask * 255).to(torch.bool) im_mask = np.zeros((im_h, im_w)) # im_mask = torch.zeros((im_h, im_w), dtype=torch.bool) x_0 = max(box[0], 0) x_1 = min(box[2] + 1, im_w) y_0 = max(box[1], 0) y_1 = min(box[3] + 1, im_h) im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]), (x_0 - box[0]):(x_1 - box[0])] del mask return im_mask
def forward(self, masks, boxes, feature_map, padding=1): assert self.cfg.MODEL.PANOPTIC.PS_ON == True for mask, box in zip(masks.unsqueeze(1), boxes.bbox): box = box.numpy().astype(np.int32) _, _, im_h, im_w = feature_map.size() TO_REMOVE = 1 w = box[2] - box[0] + TO_REMOVE h = box[3] - box[1] + TO_REMOVE w = max(w, 1) h = max(h, 1) resized_mask = interpolate(mask, [w, h], mode='bilinear') x_0 = box[0] x_1 = min(box[2] + 1, im_w) y_0 = box[1] y_1 = min(box[3] + 1, im_h) mask_cropped = resized_mask[:, :, 0:min(w, im_w - y_0), 0:min(h, im_h - x_0)] if not self.cfg.MODEL.PANOPTIC.ATTENTION_FUSION: feature_map[:, :, y_0:min(y_0 + w, im_w), x_0:min(x_0 + h, im_h)] = mask_cropped else: mask_cropped_att = mask_cropped.sigmoid().detach() feature_map[:, :, y_0:min(y_0 + w, im_w), x_0:min(x_0 + h, im_h)] *= (1 + mask_cropped_att) if not self.cfg.MODEL.PANOPTIC.ATTENTION_FUSION: out_sem_feature_fusion = feature_map else: out_sem_feature_fusion = self.sem_block_3x3(F.relu(feature_map)) out_sem_feature_fusion = self.sem_block_1x1( F.relu(out_sem_feature_fusion)) return out_sem_feature_fusion
def __init__(self, masks, size): """ Arguments: masks: Either torch.tensor of [num_instances, H, W] or list of torch.tensors of [H, W] with num_instances elems, or RLE (Run Length Encoding) - interpreted as list of dicts, or BinaryMaskList. size: absolute image size, width first After initialization, a hard copy will be made, to leave the initializing source data intact. """ assert isinstance(size, (list, tuple)) assert len(size) == 2 if isinstance(masks, torch.Tensor): # The raw data representation is passed as argument masks = masks.clone() elif isinstance(masks, (list, tuple)): if len(masks) == 0: masks = torch.empty([0, size[1], size[0]]) # num_instances = 0! elif isinstance(masks[0], torch.Tensor): masks = torch.stack(masks, dim=2).clone() elif isinstance(masks[0], dict) and "counts" in masks[0]: # RLE interpretation rle_sizes = [tuple(inst["size"]) for inst in masks] masks = mask_utils.decode(masks) # [h, w, n] masks = torch.tensor(masks).permute(2, 0, 1) # [n, h, w] assert rle_sizes.count(rle_sizes[0]) == len(rle_sizes), ( "All the sizes must be the same size: %s" % rle_sizes) # in RLE, height come first in "size" rle_height, rle_width = rle_sizes[0] assert masks.shape[1] == rle_height assert masks.shape[2] == rle_width width, height = size if width != rle_width or height != rle_height: masks = interpolate( input=masks[None].float(), size=(height, width), mode="bilinear", align_corners=False, )[0].type_as(masks) else: RuntimeError( "Type of `masks[0]` could not be interpreted: %s" % type(masks)) elif isinstance(masks, BinaryMaskList): # just hard copy the BinaryMaskList instance's underlying data masks = masks.masks.clone() else: RuntimeError( "Type of `masks` argument could not be interpreted:%s" % type(masks)) if len(masks.shape) == 2: # if only a single instance mask is passed masks = masks[None] assert len(masks.shape) == 3 assert masks.shape[1] == size[1], "%s != %s" % (masks.shape[1], size[1]) assert masks.shape[2] == size[0], "%s != %s" % (masks.shape[2], size[0]) self.masks = masks self.size = tuple(size)
def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1, do_CRF=False, crf_postprocessor=None, img_single=None, score=None, upscale=1.0, img_id=0): # Need to work on the CPU, where fp16 isn't supported - cast to float to avoid this # print("box score", score) mask = mask.float() CRF_normalize = True CRF_zoom = 3.0 CRF_zoom_th = 20000 dataset = 'ss' if dataset == 'city': print("CITYSCAPES!!!") # if CRF_normalize: # mask = (mask - mask.min()) / mask.max() # # if upscale != 1.0: # mask = torch.clamp(mask*upscale+0.5, min=0, max=1) box = box.float() padded_mask, scale = expand_masks(mask[None], padding=padding) mask = padded_mask[0, 0] box = expand_boxes(box[None], scale)[0] box = box.to(dtype=torch.int32) box[0] = max(box[0], 0) box[1] = max(box[1], 0) box[2] = min(box[2], im_w) box[3] = min(box[3], im_h) # print(im_w, im_h) TO_REMOVE = 1 w = int(box[2] - box[0] + TO_REMOVE) h = int(box[3] - box[1] + TO_REMOVE) w = max(w, 1) h = max(h, 1) # Set shape to [batchxCxHxW] mask = mask.expand((1, 1, -1, -1)) # Resize mask mask = mask.to(torch.float32) mask = interpolate(mask, size=(h, w), mode='bilinear', align_corners=False) mask = mask[0][0] # print(torch.max(mask)) if upscale != 1.0: box_size = w * h if box_size > 100: upscale = 0.15 # elif box_size > 40*40: # upscale = 0.2 else: upscale = 0.2 # print(upscale) threshold = sorted(mask.flatten(), reverse=True)[int( box_size * upscale)] + 1e-8 mask = mask / threshold mask = torch.clamp(mask, 0, 1) # CRF here!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if do_CRF and score > 0.0: img_single = img_single[:, :, ::-1] h_orig, w_orig = img_single.shape[:2] im_mask = torch.zeros((im_h, im_w)) x_0 = max(box[0], 0) x_1 = min(box[2] + 1, im_w) y_0 = max(box[1], 0) y_1 = min(box[3] + 1, im_h) # print(x_1, y_1, box[2], box[3]) p_box = [int(x_0), int(y_0), int(x_1), int(y_1)] im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]), (x_0 - box[0]):(x_1 - box[0])] # projection im_mask = im_mask.numpy() if dataset == 'city': img_crop, cut_point_crop, img_resize, p_box_crop, mask_crop, resize_ratio = crop_city_roi( p_box, img_single, mask=im_mask, return_ratio=True) # img_crop_norm = norm_image(img_crop) h_new, w_new = img_resize.shape[:2] mask_for_crf = np.concatenate([ np.expand_dims(mask_crop, 0), np.expand_dims(1 - mask_crop, 0) ], axis=0) mask_for_crf = crf_postprocessor(img_crop, mask_for_crf)[0] # mask_for_crf_norm = crf_postprocessor(img_crop_norm, mask_for_crf)[0] # cv2.imshow('img_crop', cv2.resize(img_crop, dsize=None, fx=0.5, fy=0.5)) # cv2.imshow('img_crop_norm', cv2.resize(img_crop_norm, dsize=None, fx=0.5, fy=0.5)) # # cv2.imshow('mask_crop', cv2.resize((mask_crop * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5)) # cv2.imshow('crf_crop', cv2.resize((mask_for_crf_nonorm * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5)) # cv2.imshow('crf_crop_norm', cv2.resize((mask_for_crf_norm * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5)) # mask_for_crf = mask_for_crf_nonorm total_mask = np.zeros([h_new, w_new]) total_mask[cut_point_crop[1]:cut_point_crop[3], cut_point_crop[0]:cut_point_crop[2]] = mask_for_crf mask_for_crf = cv2.resize(total_mask, (2048, 1024)) # cv2.imshow('img', cv2.resize(img_single, dsize=None, fx=0.5, fy=0.5)) # cv2.imshow('mask', cv2.resize((im_mask * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5)) # cv2.imshow('crf', cv2.resize((mask_for_crf * 255.0).astype('uint8'), dsize=None, fx=0.5, fy=0.5)) # cv2.waitKey(0) else: if CRF_zoom != 1.0 and box_size < CRF_zoom_th: cut_point = [ int(box[0] * (1 - 1 / CRF_zoom)), int(box[1] * (1 - 1 / CRF_zoom)), int((w_orig + (CRF_zoom - 1) * box[2]) / CRF_zoom), int((h_orig + (CRF_zoom - 1) * box[3]) / CRF_zoom) ] before_zoom_size = (cut_point[2] - cut_point[0], cut_point[3] - cut_point[1]) image_zoom = img_single[cut_point[1]:cut_point[3], cut_point[0]:cut_point[2]] mask_zoom = im_mask[cut_point[1]:cut_point[3], cut_point[0]:cut_point[2]] image_zoom = cv2.resize(image_zoom, (w_orig, h_orig)) mask_zoom = cv2.resize(mask_zoom, (w_orig, h_orig)) mask_for_crf = np.concatenate([ np.expand_dims(mask_zoom, 0), np.expand_dims(1 - mask_zoom, 0) ], axis=0) mask_for_crf = crf_postprocessor(image_zoom, mask_for_crf)[0] mask_zoom = cv2.resize(mask_for_crf.copy(), before_zoom_size) mask_for_crf = np.zeros(im_mask.shape) # print(w_orig, h_orig) # print(mask_zoom.shape) mask_for_crf[cut_point[1]:cut_point[3], cut_point[0]:cut_point[2]] = mask_zoom save_ratio = 1.0 else: mask_for_crf = np.concatenate([ np.expand_dims(im_mask, 0), np.expand_dims(1 - im_mask, 0) ], axis=0) mask_for_crf = crf_postprocessor(img_single, mask_for_crf)[0] # cv2.imshow('img', img_single) # # cv2.imshow('mask', (im_mask * 255.0).astype('uint8')) # cv2.imshow('mask_crf_nozoom', (mask_for_crf * 255.0).astype('uint8')) # # cv2.waitKey(0) mask = torch.Tensor(mask_for_crf) if thresh >= 0: im_mask = mask > thresh else: # for visualization and debugging, we also # allow it to return an unmodified mask im_mask = (mask * 255).to(torch.bool) else: if thresh >= 0: mask = mask > thresh else: # for visualization and debugging, we also # allow it to return an unmodified mask mask = (mask * 255).to(torch.bool) im_mask = torch.zeros((im_h, im_w), dtype=torch.bool) x_0 = max(box[0], 0) x_1 = min(box[2] + 1, im_w) y_0 = max(box[1], 0) y_1 = min(box[3] + 1, im_h) im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]):(y_1 - box[1]), (x_0 - box[0]):(x_1 - box[0])] return im_mask