def visualize_image_with_mask(line_id):
'''
Function to visualize the image and the mask.
INPUT:
line_id - id of the line to visualize the masks
'''
# replace null values with '-1'
im_df = train_df.fillna('-1')
# get segmentation mask
np_mask = get_mask(line_id)
# open the image
image = Image.open(TRAIN_PATH + im_df.loc[line_id]['Image'])
# create segmentation map
segmap = SegmentationMapOnImage(np_mask, np_mask.shape, nb_classes=2)
# visualize the image and map
side_by_side = np.hstack([segmap.draw_on_image(np.asarray(image))
]).reshape(np.asarray(image).shape)
fig, ax = plt.subplots(figsize=(6, 4))
ax.axis('off')
plt.title(im_df.loc[line_id]['Label'])
ax.imshow(side_by_side)
def apply_transform(self, im, mask):
if self.transform is not None:
mask = SegmentationMapOnImage(mask.astype(bool), im.shape)
im, mask = self.transform(image=im, segmentation_maps=mask)
mask = mask.get_arr_int() * 255
return im, mask
def __getitem__(self, index):
batch_urls = self.img_set[index * self.batch_size:(index + 1) *
self.batch_size]
batch_images = []
batch_labels = []
det = self.seq.to_deterministic()
for url in batch_urls:
current_image = np.array(
Image.open(os.path.join(self.dir_path, 'image', url + '.png')))
current_image = det.augment_image(current_image)
batch_images.append(current_image)
current_label = np.array(
Image.open(
os.path.join(self.dir_path, 'annotation',
url + '_label.png'))) // 60
current_label = SegmentationMapOnImage(current_label,
shape=current_label.shape,
nb_classes=4)
current_label = det.augment_segmentation_maps(current_label)
current_label = current_label.get_arr_int()
batch_labels.append(current_label)
batch_images = np.array(batch_images)[:, :, :, :3] / 255.
batch_labels = np.array(to_categorical(batch_labels, num_classes=4))
# det = self.seq.to_deterministic()
# batch_images = det.augment_image(batch_images)
# batch_labels = det.augment_segmentation_maps(batch_labels)
# batch_images, batch_labels = self.seq(image=batch_images, segmentation_maps=batch_labels)
return batch_images, batch_labels
def normalize_segmentation_maps(inputs, shapes=None):
# TODO get rid of this deferred import
from imgaug.augmentables.segmaps import SegmentationMapOnImage
shapes = _preprocess_shapes(shapes)
ntype = estimate_segmaps_norm_type(inputs)
_assert_exactly_n_shapes_partial = functools.partial(
_assert_exactly_n_shapes,
from_ntype=ntype,
to_ntype="List[SegmentationMapOnImage]",
shapes=shapes)
if ntype == "None":
return None
elif ntype in ["array[int]", "array[uint]", "array[bool]"]:
_assert_single_array_ndim(inputs, 3, "(N,H,W)",
"SegmentationMapOnImage")
_assert_exactly_n_shapes_partial(n=len(inputs))
if ntype == "array[bool]":
return [
SegmentationMapOnImage(attr_i, shape=shape)
for attr_i, shape in zip(inputs, shapes)
]
return [
SegmentationMapOnImage(attr_i,
shape=shape,
nb_classes=1 + np.max(attr_i))
for attr_i, shape in zip(inputs, shapes)
]
elif ntype == "SegmentationMapOnImage":
return [inputs]
elif ntype == "iterable[empty]":
return None
elif ntype in [
"iterable-array[int]", "iterable-array[uint]",
"iterable-array[bool]"
]:
_assert_many_arrays_ndim(inputs, 2, "(H,W)", "SegmentationMapsOnImage")
_assert_exactly_n_shapes_partial(n=len(inputs))
if ntype == "iterable-array[bool]":
return [
SegmentationMapOnImage(attr_i, shape=shape)
for attr_i, shape in zip(inputs, shapes)
]
return [
SegmentationMapOnImage(attr_i,
shape=shape,
nb_classes=1 + np.max(attr_i))
for attr_i, shape in zip(inputs, shapes)
]
else:
assert ntype == "iterable-SegmentationMapOnImage", (
"Got unknown normalization type '%s'." % (ntype, ))
return inputs # len allowed to differ from len of images
def augment_with_segmap(image, segmap, num_classes):
if random.random() < 0.3:
return image, segmap
segmap = SegmentationMapOnImage(segmap,
shape=image.shape,
nb_classes=num_classes)
image_aug, segmap_aug = seq(image=image, segmentation_maps=segmap)
return image_aug, None
def preprocessing(image, mask):
image, mask = random_scale_fn(image, mask)
if iaa_aug is not None:
image, segmap = iaa_aug(image=image,
segmentation_maps=SegmentationMapOnImage(
mask, shape=mask.shape, nb_classes=19))
mask = segmap.get_arr_int()
image, mask = random_rotate_crop_fn(image, mask)
return image, mask
def augmentaion(img, lab, mask, times):
# rotate
if times < 1:
return [], [], []
lab[lab > 0] = 1
lab = SegmentationMapOnImage(lab, shape=img.shape, nb_classes=2)
if mask is not None:
mask[mask > 0] = 1
mask = SegmentationMapOnImage(mask, shape=img.shape, nb_classes=2)
a_imgs = []
a_labs = []
a_masks = []
sometimes = lambda aug: iaa.Sometimes(0.8, aug)
seq = iaa.Sequential(
[
iaa.Fliplr(p=0.5),
iaa.Flipud(p=0.5),
# iaa.ElasticTransformation(alpha=(10.0,40.0), sigma=(4.0,5.0)),
sometimes(iaa.ContrastNormalization((0.5, 2.0))),
sometimes(iaa.PerspectiveTransform(scale=(0.02, 0.1)))
],
random_order=True)
for _ in range(times):
_aug = seq._to_deterministic()
aug_img = _aug.augment_image(img)
aug_lab = _aug.augment_segmentation_maps(lab).get_arr_int()
aug_lab[aug_lab > 0] = 255
a_imgs.append(aug_img)
a_labs.append(aug_lab)
if mask is not None:
aug_mask = _aug.augment_segmentation_maps(mask).get_arr_int()
aug_mask[aug_mask > 0] = 255
a_masks.append(aug_mask)
# a_imgs = aug_imgs[0:, 0, ...]
# a_labs = aug_imgs[0:, 1, ...]
return a_imgs, a_labs, a_masks
def get_random(self):
bg = self.backgrounds.get_random()
bg = self.resize_bg.augment_image(bg)
while True:
card1, card_name1, hull1 = self.cards.get_random()
card_aug1, kpsoi_aug1, segmap_aug1 = self.augment_card(
card1, hull1)
card_mask1 = self.kps_to_mask(kpsoi_aug1)
card2, card_name2, hull2 = self.cards.get_random()
card_aug2, kpsoi_aug2, segmap_aug2 = self.augment_card(
card2, hull2)
card_mask2 = self.kps_to_mask(kpsoi_aug2)
# Handle superposition
arr = segmap_aug1.get_arr_int()
original_size = np.sum(arr)
arr = np.where(card_mask2[:, :, 0], 0, arr)
segmap_aug1 = SegmentationMapOnImage(arr,
nb_classes=2,
shape=bg.shape)
new_size = np.sum(arr)
final = np.where(card_mask1, card_aug1, bg)
final = np.where(card_mask2, card_aug2, final)
if new_size < original_size * self.ratio:
continue
break
# create empty segmentation map for classes: background, tree, chipmunk
mask_final = np.zeros((imgH, imgW, 2), dtype=np.uint8)
mask_final[:, :, 0] = segmap_aug1.get_arr_int()
mask_final[:, :, 1] = segmap_aug2.get_arr_int()
# Add classes
class_id = np.array([
self.labels.index(card_name1) + 1,
self.labels.index(card_name2) + 1
])
return final, (mask_final, class_id)
def do_imgaug(num):
ia.seed(1)
new_id = 1000
for id_ in tqdm(range(num), total=num):
new_id += 1
id_ = id_ % (len(raw_img_name)-1)
image = imread(RAW_IMG_PATH + raw_img_name[id_])
segmap = imread(RAW_MASK_PATH + raw_mask_name[id_])
segmap = SegmentationMapOnImage(segmap, nb_classes=2, shape=image.shape)
seq_det = seq.to_deterministic()
imsave(AUG_IMG_PATH + str(new_id) + ".png", seq_det.augment_image(image))
imsave(AUG_MASK_PATH + str(new_id) + ".png", seq_det.augment_segmentation_maps(segmap).get_arr())
def augment_card(self, card, hull):
cardH, cardW, _ = card.shape
decalX = int((self.imgW - cardW) / 2)
decalY = int((self.imgH - cardH) / 2)
img_card = np.zeros((self.imgH, self.imgW, 4), dtype=np.uint8)
img_card[decalY:decalY + cardH, decalX:decalX + cardW, :] = card
card = img_card[:, :, :3]
hull = hull[:, 0, :]
card_kps_xy = np.float32([[decalX, decalY], [decalX + cardW, decalY],
[decalX + cardW, decalY + cardH],
[decalX, decalY + cardH]])
kpsoi_card = KeypointsOnImage.from_xy_array(card_kps_xy,
shape=card.shape)
# hull is a cv2.Contour, shape : Nx1x2
kpsoi_hull = [
ia.Keypoint(x=p[0] + decalX, y=p[1] + decalY)
for p in hull.reshape(-1, 2)
]
kpsoi_hull = KeypointsOnImage(kpsoi_hull,
shape=(self.imgH, self.imgW, 3))
# create polygon
poly_hull = Polygon(kpsoi_hull.keypoints)
# create empty segmentation map for classes: background and card
segmap = np.zeros((card.shape[0], card.shape[1], 3), dtype=np.uint8)
# draw the tree polygon into the second channel
segmap = poly_hull.draw_on_image(segmap,
color=(0, 255, 0),
alpha=1.0,
alpha_lines=0.0,
alpha_points=0.0)
# merge the two channels to a single one
segmap = np.argmax(segmap, axis=2)
segmap = segmap.astype(np.uint8)
segmap = SegmentationMapOnImage(segmap, nb_classes=2, shape=card.shape)
myseq = self.seq.to_deterministic()
card_aug, segmap_aug = myseq(image=card, segmentation_maps=segmap)
card_aug, kpsoi_aug = myseq(image=card, keypoints=kpsoi_card)
return card_aug, kpsoi_aug, segmap_aug
def put_object_onto_background(self,
img_obj, # image of the object
mask_obj, # mask of the object
img_bg, # bgd -> background
):
''' Put object onto background image with random transformations specified by args '''
img_obj = img_obj.copy()
mask_obj = mask_obj.copy()
img_bg = img_bg.copy()
new_bg, new_mask = self._put_object_onto_background(img_obj, mask_obj, img_bg)
# Augment again to the big image
# (Not use this: This causes problem when adding several objects.)
if 0:
segmap = SegmentationMapOnImage(new_mask, nb_classes=2, shape=new_bg.shape)
new_bg, new_mask = self.seq(image=new_bg, segmentation_maps=segmap)
new_mask = new_mask.get_arr_int().astype(np.uint8)
return new_bg, new_mask
def create_segmap(image_id):
'''
Helper function to create a segmentation map for an image by image filename
'''
# open the image
image = np.asarray(Image.open(image_id))
# get masks for different classes
fish_mask = get_mask_by_image_id(image_id, 'Fish')
flower_mask = get_mask_by_image_id(image_id, 'Flower')
gravel_mask = get_mask_by_image_id(image_id, 'Gravel')
sugar_mask = get_mask_by_image_id(image_id, 'Sugar')
# label numpy map with 4 classes
segmap = np.zeros((image.shape[0], image.shape[1]), dtype=np.int32)
segmap = np.where(fish_mask == 1, 1, segmap)
segmap = np.where(flower_mask == 1, 2, segmap)
segmap = np.where(gravel_mask == 1, 3, segmap)
segmap = np.where(sugar_mask == 1, 4, segmap)
# create a segmantation map
segmap = SegmentationMapOnImage(segmap, shape=image.shape, nb_classes=5)
return segmap
def _augment_seg(img, seg):
if seq[0] is None:
load_aug()
if params.augType == 'sto':
# stohastic augmentation
aug_sto = seq[0]
images = []
labels = []
image_aug = aug_sto.augment_image( img )
seg_aug = SegmentationMapOnImage(seg , nb_classes=np.max(seg)+1 , shape=img.shape)
segmap_aug = aug_sto.augment_segmentation_maps( seg_aug )
segmap_aug = segmap_aug.get_arr_int(background_class_id=3)
return image_aug, segmap_aug
if params.augType == 'det':
# deterministic augmentation
aug_det = seq[0].to_deterministic()
images = []
labels = []
image_aug = aug_det.augment_image( img )
def plot_training_images_and_masks(n_images=3):
'''
Function to plot several random images with segmentation masks.
INPUT:
n_images - number of images to visualize
'''
# get a list of images from training set
images = sorted(glob(TRAIN_PATH + '*.jpg'))
fig, ax = plt.subplots(n_images, 4, figsize=(20, 10))
# create a list of random indices
rnd_indices = [
np.random.choice(range(0, len(images))) for i in range(n_images)
]
for im in range(0, n_images):
# open image with a random index
image = Image.open(images[rnd_indices[im]])
# get segmentation masks
fish = get_mask_by_image_id(images[rnd_indices[im]], 'Fish')
flower = get_mask_by_image_id(images[rnd_indices[im]], 'Flower')
gravel = get_mask_by_image_id(images[rnd_indices[im]], 'Gravel')
sugar = get_mask_by_image_id(images[rnd_indices[im]], 'Sugar')
# draw masks on images
shape = (np.asarray(image).shape[0], np.asarray(image).shape[1])
if np.sum(fish) > 0:
segmap_fish = SegmentationMapOnImage(fish,
shape=shape,
nb_classes=2)
im_fish = np.array(segmap_fish.draw_on_image(
np.asarray(image))).reshape(np.asarray(image).shape)
else:
im_fish = np.asarray(image)
if np.sum(flower) > 0:
segmap_flower = SegmentationMapOnImage(flower,
shape=shape,
nb_classes=2)
im_flower = np.array(segmap_flower.draw_on_image(
np.asarray(image))).reshape(np.asarray(image).shape)
else:
im_flower = np.asarray(image)
if np.sum(gravel) > 0:
segmap_gravel = SegmentationMapOnImage(gravel,
shape=shape,
nb_classes=2)
im_gravel = np.array(segmap_gravel.draw_on_image(
np.asarray(image))).reshape(np.asarray(image).shape)
else:
im_gravel = np.asarray(image)
if np.sum(sugar) > 0:
segmap_sugar = SegmentationMapOnImage(sugar,
shape=shape,
nb_classes=2)
im_sugar = np.array(segmap_sugar.draw_on_image(
np.asarray(image))).reshape(np.asarray(image).shape)
else:
im_sugar = np.asarray(image)
# plot images and masks
ax[im, 0].imshow(im_fish)
ax[im, 0].axis('off')
ax[im, 0].set_title('Fish')
# plot images and masks
ax[im, 1].imshow(im_flower)
ax[im, 1].axis('off')
ax[im, 1].set_title('Flower')
# plot images and masks
ax[im, 2].imshow(im_gravel)
ax[im, 2].axis('off')
ax[im, 2].set_title('Gravel')
# plot images and masks
ax[im, 3].imshow(im_sugar)
ax[im, 3].axis('off')
ax[im, 3].set_title('Sugar')
plt.suptitle('Sample images from the train set')
plt.show()
def __getitem__(self, idx):
image_id = self.sample_list[idx]
info = self.image_info[image_id]
img_path = info["image_path"]
img = Image.open(img_path + '.png').convert("RGB")
width, height = img.size
img = img.resize((self.width, self.height), resample=Image.BILINEAR)
boxes = []
masks = []
for anno in info['annotations']:
mask = rle2mask(anno, width, height).T
mask = Image.fromarray(mask)
mask = mask.resize((self.width, self.height),
resample=Image.BILINEAR)
mask = np.expand_dims(mask, axis=0)
pos = np.where(np.array(mask)[0, :, :])
xmin = np.min(pos[1])
xmax = np.max(pos[1])
ymin = np.min(pos[0])
ymax = np.max(pos[0])
'''
xmin = np.min(pos[0])
xmax = np.max(pos[0])
ymin = np.min(pos[1])
ymax = np.max(pos[1])
'''
box = [xmin, ymin, xmax, ymax]
boxes.append(box)
masks.append(mask.squeeze())
img = np.asarray(img)
if self.aug is not None:
if len(masks) > 0:
bbs = BoundingBoxesOnImage([
BoundingBox(box[0], box[1], box[2], box[3])
for box in boxes
],
shape=img.shape)
segs = [
SegmentationMapOnImage(np.expand_dims(mask, axis=2),
shape=img.shape,
nb_classes=2) for mask in masks
]
aug_det = self.aug.to_deterministic()
aug_img = aug_det.augment_image(img)
aug_bbs = aug_det.augment_bounding_boxes(
bbs).clip_out_of_image()
aug_segs = [
aug_det.augment_segmentation_maps(seg) for seg in segs
]
# if all bboxes are moved out of image
# then we just use the original sample
if len(aug_bbs.to_xyxy_array()) > 0:
_img = aug_img.copy()
# boxes = aug_bbs.to_xyxy_array()
_masks = [
aug_seg.get_arr_int() * 255 for aug_seg in aug_segs
]
# recreate bboxes from masks to keep tight
_boxes = []
for _mask in _masks:
_mask = np.expand_dims(_mask, axis=0)
pos = np.where(np.array(_mask)[0, :, :])
# double check
if len(pos[1]) > 0:
xmin = np.min(pos[1])
xmax = np.max(pos[1])
ymin = np.min(pos[0])
ymax = np.max(pos[0])
box = [xmin, ymin, xmax, ymax]
_boxes.append(box)
# the last check
if len(_boxes) > 0:
img = _img
boxes = _boxes
masks = _masks
boxes = torch.as_tensor(boxes, dtype=torch.float32)
masks = torch.as_tensor(np.stack(masks, axis=0), dtype=torch.float32)
areas = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
labels = torch.ones((len(areas), ), dtype=torch.int64)
image_id = torch.tensor([idx])
iscrowd = torch.zeros((len(areas), ), dtype=torch.int64)
target = {}
target["boxes"] = boxes
target["labels"] = labels
target["masks"] = masks / 255
target["image_id"] = image_id
target["area"] = areas
target["iscrowd"] = iscrowd
return transforms.ToTensor()(img), target
def get_segmentation_map(self, segmap, nb_classes, shape):
return SegmentationMapOnImage(segmap,
nb_classes=nb_classes,
shape=shape)
image_list = os.listdir(file_path)
#mask_list = [image for image in image_list if image.split('.')[1] == 'png']
image_list = sorted([image for image in image_list if image.split('.')[1]!='png'])
image_list = image_list[1:]
#image_list = random.sample(image_list, k=10)
for img in image_list:
# Load image and mask
mask_path = os.path.join('./mask', img.split('.')[0] + '_color_mask.png')
image = skimage.io.imread(os.path.join(file_path,img))
h,w,_ = image.shape
image = np.pad(image, ((100,100), (100,100), (0, 0)), mode='constant')
segmap = skimage.io.imread(mask_path).astype('float32')
segmap = np.pad(segmap, ((100,100), (100,100), (0, 0)), mode='constant')
segmap = SegmentationMapOnImage(segmap, nb_classes=3, shape=image.shape)
# Define our augmentation pipeline.
seq = iaa.Sequential([
#iaa.PadToFixedSize(height=h+100,width=w+100, position='center'),
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.AddToHueAndSaturation((-50, 50)), # change hue and saturation
#iaa.Dropout([0.05, 0.2]), # drop 5% or 20% of all pixels
#iaa.Sharpen((0.0, 1.0)), # sharpen the image
iaa.Affine(rotate=(-90, 90)), # rotate by -30 to 30 degrees (affects segmaps)
#iaa.ElasticTransformation(alpha=50, sigma=5) # apply water effect (affects segmaps)
iaa.PerspectiveTransform(scale=(0.04, 0.1)),
iaa.PiecewiseAffine(scale=(0.04, 0.05))
], random_order=False)
