def __getitem__(self, index):
'''Returns an item from the dataset at the given index. If no labels directory has been specified,
then a tensor of zeroes will be returned as the label.
Args:
index (int): index of the item required from dataset.
Returns:
torch.Tensor: Tensor of input image
torch.Tensor: Tensor of label (Tensor of zeroes is labels_dir is "" or None)
'''
# Open input imgs
image_path = self._datalist_input[index]
_img = Image.open(image_path).convert('RGB')
_img = np.array(_img)
# Open labels
if self.labels_dir:
label_path = self._datalist_label[index]
_label = exr_loader(label_path, ndim=3) # (3, H, W)
if self.masks_dir:
mask_path = self._datalist_mask[index]
_mask = imageio.imread(mask_path)
# Apply image augmentations and convert to Tensor
if self.transform:
det_tf = self.transform.to_deterministic()
_img = det_tf.augment_image(_img)
if self.labels_dir:
# Making all values of invalid pixels marked as -1.0 to 0.
# In raw data, invalid pixels are marked as (-1, -1, -1) so that on conversion to RGB they appear black.
mask = np.all(_label == -1.0, axis=0)
_label[:, mask] = 0.0
_label = _label.transpose((1, 2, 0)) # To Shape: (H, W, 3)
_label = det_tf.augment_image(_label, hooks=ia.HooksImages(activator=self._activator_masks))
_label = _label.transpose((2, 0, 1)) # To Shape: (3, H, W)
if self.masks_dir:
_mask = det_tf.augment_image(_mask, hooks=ia.HooksImages(activator=self._activator_masks))
# Return Tensors
_img_tensor = transforms.ToTensor()(_img)
if self.labels_dir:
_label_tensor = torch.from_numpy(_label)
_label_tensor = nn.functional.normalize(_label_tensor, p=2, dim=0)
else:
_label_tensor = torch.zeros((3, _img_tensor.shape[1], _img_tensor.shape[2]), dtype=torch.float32)
if self.masks_dir:
_mask = _mask[..., np.newaxis]
_mask_tensor = transforms.ToTensor()(_mask)
else:
_mask_tensor = torch.ones((1, _img_tensor.shape[1], _img_tensor.shape[2]), dtype=torch.float32)
return _img_tensor, _label_tensor, _mask_tensor
def data_augment_volume(self, *datalist, augmentation):
# first get the volume data from the data list
image1, image2, image3, mask1, mask2, mask3 = datalist
# Augmentation
# This requires the imgaug lib (https://github.com/aleju/imgaug)
if augmentation:
import imgaug
# Augmenters that are safe to apply to masks
# Some, such as Affine, have settings that make them unsafe, so always
# test your augmentation on masks
MASK_AUGMENTERS = [
"Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr",
"Flipud", "CropAndPad", "Affine", "PiecewiseAffine"
]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return augmenter.__class__.__name__ in MASK_AUGMENTERS
# Store shapes before augmentation to compare
image1_shape = image1.shape
mask1_shape = mask1.shape
image2_shape = image2.shape
mask2_shape = mask2.shape
image3_shape = image3.shape
mask3_shape = mask3.shape
# Make augmenters deterministic to apply similarly to images and masks
det = augmentation.to_deterministic()
# image should be uint8!!
image1 = det.augment_image(image1)
image2 = det.augment_image(image2)
image3 = det.augment_image(image3)
# Change mask to np.uint8 because imgaug doesn't support np.bool
mask1 = det.augment_image(mask1.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
mask2 = det.augment_image(mask2.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
mask3 = det.augment_image(mask3.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
# Verify that shapes didn't change
assert image1.shape == image1_shape, "Augmentation shouldn't change image size"
assert mask1.shape == mask1_shape, "Augmentation shouldn't change mask size"
assert image2.shape == image2_shape, "Augmentation shouldn't change image size"
assert mask2.shape == mask2_shape, "Augmentation shouldn't change mask size"
assert image3.shape == image3_shape, "Augmentation shouldn't change image size"
assert mask3.shape == mask3_shape, "Augmentation shouldn't change mask size"
# Change mask back to bool
# masks = masks.astype(np.bool)
return image1, image2, image3, mask1, mask2, mask3
def _augmentation(self, image, mask):
# Augmenters that are safe to apply to masks
# Some, such as Affine, have settings that make them unsafe, so always
# test your augmentation on masks
MASK_AUGMENTERS = ["Sequential", "SomeOf", "OneOf", "Sometimes",
"Fliplr", "Flipud", "CropAndPad",
"Affine", "PiecewiseAffine"]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return augmenter.__class__.__name__ in MASK_AUGMENTERS
# Store shapes before augmentation to compare
image_shape = image.shape
mask_shape = mask.shape
# Make augmenters deterministic to apply similarly to images and masks
det = self.transform.to_deterministic()
image = det.augment_image(image)
# Change mask to np.uint8 because imgaug doesn't support np.bool
mask = det.augment_image(mask.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
image = np.array(self._color_jitter(Image.fromarray(image)))
# Verify that shapes didn't change
assert image.shape == image_shape, "Augmentation shouldn't change image size"
assert mask.shape == mask_shape, "Augmentation shouldn't change mask size"
# Change mask back to bool
# label = label.astype(np.bool)
return image, mask
def image_aug(self, image, mask):
hooks_binmasks = ia.HooksImages(activator=self.activator_binmasks)
seq = iaa.SomeOf((1, 2), [
iaa.OneOf([
iaa.Affine(rotate=(-30, 30), name="Rotate"),
iaa.Affine(scale=(0.3, 1.3), name="Scale")
]),
iaa.OneOf([
iaa.Multiply((0.5, 1.5), name="Multiply"),
iaa.GaussianBlur((0, 3.0), name="GaussianBlur"),
iaa.CoarseDropout((0.05, 0.2),
size_percent=(0.01, 0.1),
name="CoarseDropout")
])
])
seq_det = seq.to_deterministic(
) # call this for each batch again, NOT only once at the start
image_aug = np.squeeze(seq_det.augment_images(
np.expand_dims(np.array(image), axis=0)),
axis=0)
mask_aug = np.squeeze(seq_det.augment_images(
(np.expand_dims(np.array(mask), axis=0)), hooks=hooks_binmasks),
axis=0)
return Image.fromarray(image_aug), Image.fromarray(mask_aug)
def __init__(self, config):
self._config = config
self._seq = iaa.Sequential([
iaa.Sometimes(config["crop_probability"], iaa.Crop(px=(
0, 16
))), # crop images from each side by 0 to 16px (randomly chosen)
iaa.Sometimes(
config["blur_probability"], iaa.GaussianBlur(
sigma=(0, 3.0))), # blur images with a sigma of 0 to 3.0,
iaa.Sometimes(
config["affine_probability"],
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-45, 45),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL))
])
# https://github.com/aleju/imgaug/issues/41
self._hooks_semantic = ia.HooksImages(
activator=self.activator_semantic)
def aug(image,mask):
augmentation = imgaug.augmenters.Fliplr(0.5)
MASK_AUGMENTERS = ["Sequential", "SomeOf", "OneOf", "Sometimes",
"Fliplr", "Flipud", "CropAndPad",
"Affine", "PiecewiseAffine"]
def hook(images, augmenter, parents, default):
return (augmenter.__class__.__name__ in MASK_AUGMENTERS)
image_shape = image.shape
mask_shape = mask.shape
det = augmentation.to_deterministic()
image = det.augment_image(image)
mask = det.augment_image(mask.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
assert image.shape == image_shape, "Augmentation shouldn't change image size"
assert mask.shape == mask_shape, "Augmentation shouldn't change mask size"
mask = mask.astype(np.bool)
return image,mask
def __data_augmentation(self, x, y, im_paths):
""" augments data and labels, if necessary. Activator is used to restrict certain augmentations only to the
images and not the masks
Args:
x (ndarray): batch of images
y (ndaray): corresponding batch of masks
im_paths (str): if augmetation should be saved, the path of the images is used for knowing name of image.
"""
def activator(images, augmenter, parents, default):
return False if augmenter.name in [
'blur', 'contrast', 'grayscale', 'brightness'
] else default
seq_det = self.seq.to_deterministic()
x_aug = seq_det.augment_images(x)
y_aug = seq_det.augment_images(
y, hooks=ia.HooksImages(activator=activator))
if self.save_to_dir is not None:
for i, p in enumerate(im_paths):
base, tail = os.path.split(p)
root, ext = os.path.splitext(tail)
save_dir = os.path.join(
self.save_to_dir, root + '_' + str(randint(0, 1000)) + ext)
save_img(save_dir, x_aug[i, ])
return x_aug, y_aug
def test_propagation_hooks_ctx(self):
def propagator(images, augmenter, parents, default):
if ia.is_np_array(images):
return False
else:
return True
hooks = ia.HooksImages(propagator=propagator)
batch = ia.BatchInAugmentation(
images=np.zeros((3, 3, 4, 1), dtype=np.uint8),
keypoints=[
ia.KeypointsOnImage([ia.Keypoint(0, 0)], shape=(3, 4, 1)),
ia.KeypointsOnImage([ia.Keypoint(1, 1)], shape=(3, 4, 1)),
ia.KeypointsOnImage([ia.Keypoint(2, 2)], shape=(3, 4, 1))
])
with batch.propagation_hooks_ctx(iaa.Identity(), hooks, []) \
as batch_prop:
assert batch_prop.images is None
assert batch_prop.keypoints is not None
assert len(batch_prop.keypoints) == 3
batch_prop.keypoints[0].keypoints[0].x = 10
assert batch.images is not None
assert batch.keypoints is not None
assert batch.keypoints[0].keypoints[0].x == 10
def load_image_gt(dataset, config, augmentation=None):
texture_num = np.random.randint(2,6)
image_data, mask_data, class_data, texture = \
dataset.generate_image_mask(config.IMAGE_MAX_DIM, texture_num,
config.TEXTURE_SIZE)
if augmentation:
import imgaug
MASK_AUGMENTERS = ["Sequential", "SomeOf", "OneOf", "Sometimes",
"Fliplr", "Flipud", "CropAndPad",
"Affine", "PiecewiseAffine"]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return (augmenter.__class__.__name__ in MASK_AUGMENTERS)
# Store shapes before augmentation to compare
image_data_shape = image_data.shape
mask_shape = mask_data.shape
# Make augmenters deterministic to apply similarly to images and masks
det = augmentation.to_deterministic()
image_data = det.augment_image(image_data.astype(np.uint8))
# Change mask to np.uint8 because imgaug doesn't support np.bool
mask_data = det.augment_image(mask_data.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
# Verify that shapes didn't change
assert image_data.shape == image_data_shape, "Augmentation shouldn't change image size"
assert mask_data.shape == mask_shape, "Augmentation shouldn't change image size"
# cv2.imwrite("./image.jpg", image_data)
# cv2.imwrite("./mask.jpg", mask_data*255)
return image_data, mask_data, class_data, texture
def aug_iamges_backgroud(images,backgrouds):
# images and heatmaps, just arrays filled with value 30
#images = np.ones((16, 128, 128, 3), dtype=np.uint8) * 30
#heatmaps = np.ones((16, 128, 128, 21), dtype=np.uint8) * 30
# add vertical lines to see the effect of flip
#images[:, 16:128-16, 120:124, :] = 120
#heatmaps[:, 16:128-16, 120:124, :] = 120
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
iaa.Fliplr(0.5, name="Flipper"),
sometimes(iaa.GaussianBlur((0, 3.0), name="GaussianBlur")),
sometimes(iaa.Add((-10, 10), per_channel=0.5, name='add')), # change brightness of images (by -10 to 10 of original value)
sometimes(iaa.AddToHueAndSaturation((-20, 20), name='addhue')), # change hue and saturation
sometimes(iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5, name='contrast')), # improve or worsen the contrast
])
# change the activated augmenters for heatmaps,
# we only want to execute horizontal flip, affine transformation and one of
# the gaussian noises
def activator_heatmaps(images, augmenter, parents, default):
if augmenter.name in ["GaussianBlur", "add", "contrast", "addhue"]:
return False
else:
# default value for all other augmenters
return default
hooks_heatmaps = ia.HooksImages(activator=activator_heatmaps)
seq_det = seq.to_deterministic() # call this for each batch again, NOT only once at the start
images_aug = seq_det.augment_images(images)
backgrouds_aug = seq_det.augment_images(backgrouds, hooks=hooks_heatmaps)
return images_aug, backgrouds_aug
def test_TGS_salt_aug_callback():
# image + mask
# from 101*101
x, y = load_sample_image()
x = x[:5]
y = y[:5]
# x = np.concatenate([x[:1] for i in range(5)])
# y = np.concatenate([y[:1] for i in range(5)])
import random
ia.seed(random.randint(1, 10000))
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# bright add
# contrast multiply
seq = iaa.Sequential([
iaa.OneOf([
iaa.PiecewiseAffine((0.002, 0.1), name='PiecewiseAffine'),
iaa.Affine(rotate=(-20, 20)),
iaa.Affine(shear=(-45, 45)),
iaa.Affine(translate_percent=(0, 0.3), mode='symmetric'),
iaa.Affine(translate_percent=(0, 0.3), mode='wrap'),
iaa.PerspectiveTransform((0.0, 0.3))
], name='affine'),
iaa.Fliplr(0.5, name="horizontal flip"),
iaa.Crop(percent=(0, 0.3), name='crop'),
# image only
iaa.OneOf([
iaa.Add((-45, 45), name='bright'),
iaa.Multiply((0.5, 1.5), name='contrast')]
),
iaa.OneOf([
iaa.AverageBlur((1, 5), name='AverageBlur'),
# iaa.BilateralBlur(),
iaa.GaussianBlur((0.1, 2), name='GaussianBlur'),
iaa.MedianBlur((1, 7), name='MedianBlur'),
], name='blur'),
# scale to 128 * 128
iaa.Scale((128, 128), name='to 128 * 128'),
])
activator = ActivatorMask(['bright', 'contrast', 'AverageBlur', 'GaussianBlur', 'MedianBlur'])
hook_func = ia.HooksImages(activator=activator)
n_iter = 5
tile = []
for idx in range(n_iter):
print(idx)
seq_det = seq.to_deterministic()
image_aug = seq_det.augment_images(x)
mask_aug = seq_det.augment_images(y, hooks=hook_func)
tile += [image_aug]
tile += [mask_aug]
tile = np.concatenate(tile)
plot.plot_image_tile(tile, title=f'test_image_aug', column=5, )
def get_trivial_augmenter():
seq = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
])
hooks_label = imgaug.HooksImages()
return seq, hooks_label
def aug_img(self, images):
if not isinstance(images, list):
images = [images]
seq = self.aug_seq()
hook_activate = ia.HooksImages(activator=self.hook)
seq_det = seq.to_deterministic(
) # call this for each batch again, NOT only once at the start
# augment images
images_aug = seq_det.augment_images(images, hooks=hook_activate)
return images_aug
def __getitem__(self, index):
'''Returns an item from the dataset at the given index. If no labels directory has been specified,
then a tensor of zeroes will be returned as the label.
Args:
index (int): index of the item required from dataset.
Returns:
torch.Tensor: Tensor of input image
torch.Tensor: Tensor of label
'''
# Open input imgs
image_path = self._datalist_input[index]
# for surface normals as input
# _img = api_utils.exr_loader(image_path, ndim=3) # (3, H, W)
# _img = (_img + 1) / 2
# _img = _img.transpose(1,2,0)
# for rgb images as input
_img = Image.open(image_path).convert('RGB')
_img = np.array(_img)
# Open labels
if self.labels_dir:
label_path = self._datalist_label[index]
# _label = Image.open(label_path).convert('L')
mask = imageio.imread(label_path)
# _label = np.array(_label)[..., np.newaxis]
_label = np.zeros(mask.shape, dtype=np.uint8)
_label[mask >= 100] = 1
# Apply image augmentations and convert to Tensor
if self.transform:
det_tf = self.transform.to_deterministic()
_img = det_tf.augment_image(_img)
_img = np.ascontiguousarray(
_img
) # To prevent errors from negative stride, as caused by fliplr()
if self.labels_dir:
_label = det_tf.augment_image(
_label, hooks=ia.HooksImages(activator=self._activator_masks))
# Return Tensors
_img_tensor = transforms.ToTensor()(_img)
if self.labels_dir:
_label_tensor = torch.from_numpy(_label.astype(np.float32))
_label_tensor = torch.unsqueeze(_label_tensor, 0)
# _label_tensor = transforms.ToTensor()(_label.astype(np.float))
else:
_label_tensor = torch.zeros(
(1, _img_tensor.shape[1], _img_tensor.shape[2]),
dtype=torch.float32)
return _img_tensor, _label_tensor
def load_image_gt(dataset, config, image_id, augment=False, augmentation=None,
use_mini_mask=False):
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding, crop)
if augment:
logging.warning("'augment' is deprecated. Use 'augmentation' instead.")
if random.randint(0, 1):
image = np.fliplr(image)
mask = np.fliplr(mask)
if augmentation:
import imgaug
MASK_AUGMENTERS = ["Sequential", "SomeOf", "OneOf", "Sometimes",
"Fliplr", "Flipud", "CropAndPad",
"Affine", "PiecewiseAffine"]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return augmenter.__class__.__name__ in MASK_AUGMENTERS
image_shape = image.shape
mask_shape = mask.shape
det = augmentation.to_deterministic()
image = det.augment_image(image)
mask = det.augment_image(mask.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
assert image.shape == image_shape, "Augmentation shouldn't change image size"
assert mask.shape == mask_shape, "Augmentation shouldn't change mask size"
mask = mask.astype(np.bool)
_idx = np.sum(mask, axis=(0, 1)) > 0
mask = mask[:, :, _idx]
class_ids = class_ids[_idx]
bbox = utils.extract_bboxes(mask)
active_class_ids = np.zeros([dataset.num_classes], dtype=np.int32)
source_class_ids = dataset.source_class_ids[dataset.image_info[image_id]["source"]]
active_class_ids[source_class_ids] = 1
if use_mini_mask:
mask = utils.minimize_mask(bbox, mask, config.MINI_MASK_SHAPE)
image_meta = compose_image_meta(image_id, original_shape, image.shape,
window, scale, active_class_ids)
return image, image_meta, class_ids, bbox, mask
def load_image_AB(dataset, config, image_id, augmentation=None):
image_front, image_right, image_left, image_up, image_down, image_linedrawing = dataset.load_image_gt(
image_id)
image_front, window, scale, padding = utils.resize_image(
image_front,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
padding=config.IMAGE_PADDING)
image_right = utils.resize_image_b(image_right, scale, padding)
image_left = utils.resize_image_b(image_left, scale, padding)
image_up = utils.resize_image_b(image_up, scale, padding)
image_down = utils.resize_image_b(image_down, scale, padding)
image_linedrawing = utils.resize_image_b(image_linedrawing, scale, padding)
if augmentation:
import imgaug
MASK_AUGMENTERS = [
"Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud",
"CropAndPad", "Affine", "PiecewiseAffine"
]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return (augmenter.__class__.__name__ in MASK_AUGMENTERS)
# Store shapes before augmentation to compare
image_front_shape = image_front.shape
image_right_shape = image_right.shape
image_left_shape = image_left.shape
image_up_shape = image_up.shape
image_down_shape = image_down.shape
image_linedrawing_shape = image_linedrawing.shape
# Make augmenters deterministic to apply similarly to images and masks
det = augmentation.to_deterministic()
image_front = det.augment_image(image_front.astype(np.uint8))
image_right = det.augment_image(image_right.astype(np.uint8))
image_left = det.augment_image(image_left.astype(np.uint8))
image_up = det.augment_image(image_up.astype(np.uint8))
image_down = det.augment_image(image_down.astype(np.uint8))
# Change mask to np.uint8 because imgaug doesn't support np.bool
image_linedrawing = det.augment_image(
image_linedrawing.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
# Verify that shapes didn't change
assert image_front_shape == image_front.shape, "Augmentation shouldn't change image size"
assert image_right_shape == image_right.shape, "Augmentation shouldn't change image size"
assert image_left_shape == image_left.shape, "Augmentation shouldn't change image size"
assert image_up_shape == image_up.shape, "Augmentation shouldn't change image size"
assert image_down_shape == image_down.shape, "Augmentation shouldn't change image size"
assert image_linedrawing_shape == image_linedrawing.shape, "Augmentation shouldn't change image size"
return image_front, image_right, image_left, image_up, image_down, image_linedrawing
def __getitem__(self, index):
'''Returns an item from the dataset at the given index. If no labels directory has been specified,
then a tensor of zeroes will be returned as the label.
Args:
index (int): index of the item required from dataset.
Returns:
torch.Tensor: Tensor of input image
torch.Tensor: Tensor of label (Tensor of zeroes is labels_dir is "" or None)
'''
# Open input imgs
image_path = self._datalist_input[index]
_img = Image.open(image_path).convert('RGB')
_img = np.array(_img)
# Open labels
if self.labels_dir:
label_path = self._datalist_label[index]
_label = utils.exr_loader(label_path, ndim=3) # (3, H, W)
# Apply image augmentations and convert to Tensor
if self.transform:
det_tf = self.transform.to_deterministic()
_img = det_tf.augment_image(_img)
# img = np.ascontiguousarray(_img) # To prevent errors from negative stride, as caused by fliplr()
if self.labels_dir:
# NOTE! EXPERIMENTAL - needs to be checked.
# covert normals into an image of dtype float32 in range [0, 1] from range [-1, 1]
_label = (_label + 1) / 2
_label = _label.transpose((1, 2, 0)) # (H, W, 3)
_label = det_tf.augment_image(
_label,
hooks=ia.HooksImages(activator=self._activator_masks))
# covert normals back to range [-1, 1]
_label = _label.transpose((2, 0, 1)) # (3, H, W)
_label = (_label * 2) - 1
# Return Tensors
_img_tensor = transforms.ToTensor()(_img)
if self.labels_dir:
_label_tensor = torch.from_numpy(_label)
else:
_label_tensor = torch.zeros(
(3, _img_tensor.shape[1], _img_tensor.shape[2]),
dtype=torch.float32)
return _img_tensor, _label_tensor
def apply_augmentation(image, mask, augmentation):
"""Apply imgaug augmentation pipeline to both a mask and an image
Reference:
- https://github.com/aleju/imgaug/issues/41
- https://github.com/matterport/Mask_RCNN/blob/4129a27275c48c672f6fd8c6303a88ba1eed643b/mrcnn/model.py
"""
import imgaug
# Augmentors that are safe to apply to masks
# Some, such as Affine, have settings that make them unsafe
MASK_AUGMENTERS = [
"Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud",
"CropAndPad", "Affine", "PiecewiseAffine"
]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return (augmenter.__class__.__name__ in MASK_AUGMENTERS)
if image.dtype != np.uint8:
raise ValueError(
'Image must be of type uint8 for augmentation (given {})'.format(
image.dtype))
if mask.dtype != np.bool:
raise ValueError(
'Mask must be of type boolean for augmentation (given {})'.format(
mask.dtype))
# Store shapes before augmentation to compare
image_shape = image.shape
mask_shape = mask.shape
# Make augmenters deterministic to apply similarly to images and masks
det = augmentation.to_deterministic()
image = det.augment_image(image)
# Change mask to np.uint8 because imgaug doesn't support np.bool
mask = det.augment_image(mask.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
# Verify that shapes didn't change
assert image.shape == image_shape, "Augmentation shouldn't change image size"
assert mask.shape == mask_shape, "Augmentation shouldn't change mask size"
# Change mask back to bool
mask = mask.astype(np.bool)
return image, mask
def example_hooks():
print("Example: Hooks")
import numpy as np
import imgaug as ia
import imgaug.augmenters as iaa
# Images and heatmaps, just arrays filled with value 30.
# We define the heatmaps here as uint8 arrays as we are going to feed them
# through the pipeline similar to normal images. In that way, every
# augmenter is applied to them.
images = np.full((16, 128, 128, 3), 30, dtype=np.uint8)
heatmaps = np.full((16, 128, 128, 21), 30, dtype=np.uint8)
# add vertical lines to see the effect of flip
images[:, 16:128 - 16, 120:124, :] = 120
heatmaps[:, 16:128 - 16, 120:124, :] = 120
seq = iaa.Sequential([
iaa.Fliplr(0.5, name="Flipper"),
iaa.GaussianBlur((0, 3.0), name="GaussianBlur"),
iaa.Dropout(0.02, name="Dropout"),
iaa.AdditiveGaussianNoise(scale=0.01 * 255, name="MyLittleNoise"),
iaa.AdditiveGaussianNoise(loc=32,
scale=0.0001 * 255,
name="SomeOtherNoise"),
iaa.Affine(translate_px={"x": (-40, 40)}, name="Affine")
])
# change the activated augmenters for heatmaps,
# we only want to execute horizontal flip, affine transformation and one of
# the gaussian noises
def activator_heatmaps(images, augmenter, parents, default):
if augmenter.name in ["GaussianBlur", "Dropout", "MyLittleNoise"]:
return False
else:
# default value for all other augmenters
return default
hooks_heatmaps = ia.HooksImages(activator=activator_heatmaps)
# call to_deterministic() once per batch, NOT only once at the start
seq_det = seq.to_deterministic()
images_aug = seq_det(images=images)
heatmaps_aug = seq_det(images=heatmaps, hooks=hooks_heatmaps)
# -----------
ia.show_grid(images_aug)
ia.show_grid(heatmaps_aug[..., 0:3])
def __getitem__(self, idx):
transform = self.transform
if idx < len(self.target_ids):
target_img = self._load_target_image(
os.path.join(self.target_root, self.target_ids[idx]))
input_img = self._load_input_image(
os.path.join(self.input_root, self.input_ids[idx]))
else:
input_img = self._load_input_image(
os.path.join(self.test_root, self.input_ids[idx]))
target_img = torch.zeros([1, 101, 101], dtype=torch.float32) - 1
transform = None
if idx < len(self.target_ids):
target_img = target_img.astype(np.uint8)
input_img = input_img.astype(np.uint8)
#This is a combined transformation for both Image and Label
if transform:
det_tf = self.transform.to_deterministic()
input_img = det_tf.augment_image(input_img)
target_img = det_tf.augment_image(
target_img,
hooks=ia.HooksImages(activator=self._activator_masks))
#npad = ( (14, 13), (14, 13),(0, 0))
#input_img = np.pad(input_img, pad_width=npad, mode='constant', constant_values=0)
to_tensor = transforms.ToTensor()
if idx < len(self.target_ids):
target_img = to_tensor(target_img)
input_img = to_tensor(input_img)
if self.norm == True:
trans = transforms.Compose([
transforms.Normalize(
mean=[102.9801 / 255, 115.9465 / 255, 122.7717 / 255],
std=[1., 1., 1.])
])
input_img = trans(input_img)
output = dict()
output['img_data'] = input_img
output['seg_label'] = target_img
return output
def __getitem__(self, index):
'''Returns an item from the dataset at the given index. If no labels directory has been specified,
then a tensor of zeroes will be returned as the label.
Args:
index (int): index of the item required from dataset.
Returns:
torch.Tensor: Tensor of input image
torch.Tensor: Tensor of label
'''
# Open input imgs
image_path = self._datalist_input[index]
_img = Image.open(image_path).convert('RGB')
_img = np.array(_img)
# Open labels
if self.labels_dir:
label_path = self._datalist_label[index]
_label = Image.open(label_path).convert('L')
_label = np.array(_label)[..., np.newaxis]
# Apply image augmentations and convert to Tensor
if self.transform:
det_tf = self.transform.to_deterministic()
_img = det_tf.augment_image(_img)
_img = np.ascontiguousarray(
_img
) # To prevent errors from negative stride, as caused by fliplr()
if self.labels_dir:
_label = det_tf.augment_image(
_label,
hooks=ia.HooksImages(activator=self._activator_masks))
# Return Tensors
_img_tensor = transforms.ToTensor()(_img)
if self.labels_dir:
_label_tensor = transforms.ToTensor()(_label.astype(np.float))
else:
_label_tensor = torch.zeros(
(1, _img_tensor.shape[1], _img_tensor.shape[2]),
dtype=torch.float32)
return _img_tensor, _label_tensor
def imgaug_augmentation(image, masks, augmentation):
import imgaug
MASK_AUGMENTERS = [
"Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud",
"CropAndPad", "Affine", "PiecewiseAffine"
]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return augmenter.__class__.__name__ in MASK_AUGMENTERS
# Make augmenters deterministic to apply similarly to images and masks
det = augmentation.to_deterministic()
image = det.augment_image(image)
# Change mask to np.uint8 because imgaug doesn't support np.bool
masks = det.augment_image(masks.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
return image, masks
def __init__(self,
images,
masks,
aug_seq,
activator,
n_batch,
n_jobs=1,
q_size=50,
verbose=0):
super().__init__(verbose)
self.images = images
self.masks = masks
self.activator = activator
self.hook_func = ia.HooksImages(activator=self.activator)
self.n_batch = n_batch
self.aug_seq = aug_seq
self.n_jobs = n_jobs
self.q_size = q_size
self.manager = mp.Manager()
self.q = self.manager.Queue(maxsize=self.q_size)
self.join_signal = mp.Event()
self.pool = None
self.workers = []
self.finished_signals = []
for _ in range(n_jobs):
finished_signal = mp.Event()
self.finished_signals += [finished_signal]
shared_images = NpSharedObj.from_np(self.images)
shared_masks = NpSharedObj.from_np(self.masks)
worker = mp.Process(
target=ImgMaskAug._iter_augment,
args=(None, shared_images.encode(), shared_masks.encode(),
self.n_batch, self.aug_seq, self.hook_func, self.q,
self.join_signal, finished_signal),
)
worker.daemon = True
worker.start()
self.log.info(f'start augmentation worker {_ + 1}/{n_jobs}')
self.workers += [worker]
def aug_img_masks(self, images, masks):
'''
images:[img1,img2]
masks:[m1,m2]
'''
if not isinstance(images, list):
images = [images]
if not isinstance(masks, list):
masks = [masks]
masks_on_images = []
for cur_img, cur_mask in zip(images, masks):
segmap = ia.SegmentationMapOnImage(cur_mask,
shape=cur_img.shape,
nb_classes=2)
masks_on_images.append(segmap)
seq = self.aug_seq()
hook_activate = ia.HooksImages(activator=self.hook)
seq_det = seq.to_deterministic(
) # call this for each batch again, NOT only once at the start
# augment masks and images
images_aug = seq_det.augment_images(images, hooks=hook_activate)
segmaps_aug = seq_det.augment_segmentation_maps(masks_on_images,
hooks=hook_activate)
img_out = []
masks_out = []
cells = []
for img_idx, (image_aug,
segmap_aug) in enumerate(zip(images_aug, segmaps_aug)):
img_out.append(image_aug) # column 3
masks_out.append(segmap_aug)
if self.DEBUG:
cells.append(images[img_idx]) # column 1
cells.append(masks_on_images[img_idx].draw_on_image(
images[img_idx])) # column 2
cells.append(image_aug) # column 3
cells.append(segmap_aug.draw_on_image(image_aug)) # column 4
cells.append(masks_on_images[img_idx].draw(
size=image_aug.shape[:2])) # column 5
# Convert cells to grid image and save
if self.DEBUG:
grid_image = ia.draw_grid(cells, cols=5)
ia.show_grid(cells, cols=5)
#imageio.imwrite("example_segmaps.jpg", grid_image)
return img_out, masks_out
def example_hooks():
print("Example: Hooks")
import imgaug as ia
#from imgaug import augmenters as iaa
import augmenters as iaa
import numpy as np
# images and heatmaps, just arrays filled with value 30
images = np.ones((16, 128, 128, 3), dtype=np.uint8) * 30
heatmaps = np.ones((16, 128, 128, 21), dtype=np.uint8) * 30
# add vertical lines to see the effect of flip
images[:, 16:128 - 16, 120:124, :] = 120
heatmaps[:, 16:128 - 16, 120:124, :] = 120
seq = iaa.Sequential([
iaa.Fliplr(0.5, name="Flipper"),
iaa.GaussianBlur((0, 3.0), name="GaussianBlur"),
iaa.Dropout(0.02, name="Dropout"),
iaa.AdditiveGaussianNoise(scale=0.01 * 255, name="MyLittleNoise"),
iaa.AdditiveGaussianNoise(loc=32,
scale=0.0001 * 255,
name="SomeOtherNoise"),
iaa.Affine(translate_px={"x": (-40, 40)}, name="Affine")
])
# change the activated augmenters for heatmaps
def activator_heatmaps(images, augmenter, parents, default):
if augmenter.name in ["GaussianBlur", "Dropout", "MyLittleNoise"]:
return False
else:
# default value for all other augmenters
return default
hooks_heatmaps = ia.HooksImages(activator=activator_heatmaps)
seq_det = seq.to_deterministic(
) # call this for each batch again, NOT only once at the start
images_aug = seq_det.augment_images(images)
heatmaps_aug = seq_det.augment_images(heatmaps, hooks=hooks_heatmaps)
# -----------
ia.show_grid(images_aug)
ia.show_grid(heatmaps_aug[..., 0:3])
def get_augmenter():
def sometimes(aug):
return iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
sometimes(
iaa.Affine(
scale={
"x": (0.9, 1.1),
"y": (0.9, 1.1)
},
rotate=(-45, 45),
shear=(-16, 16),
mode="reflect",
order=3)),
iaa.SomeOf((0, 3), [
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
sometimes(
iaa.ElasticTransformation(
alpha=(0.5, 3.5), sigma=0.25, order=3)),
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05), order=3)),
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
])
def activator_label(images, augmenter, parents, default):
if augmenter.name in [
"Sharpen", "ContrastNormalization", "AdditiveGaussianNoise",
"GaussianBlur", "AverageBlur", "MedianBlur"
]:
return False
else:
return default
hooks_label = imgaug.HooksImages(activator=activator_label)
return seq, hooks_label
def img_aug():
"""Image augmentation
Image augmentation using python iaa package
Args:
- None
Returns:
- augmenter(iaa augmenter instance): the augmenter for using
- mask_hook(iaa hook): to do the same thing with mask, we need this hook to record what augmenter had done
"""
aug = iaa.SomeOf((0, 3), [
iaa.Flipud(0.5, name="Flipud"),
iaa.Fliplr(0.5, name="Fliplr"),
iaa.AdditiveGaussianNoise(scale=0.005*255),
iaa.Grayscale(alpha=(0.0, 0.5)),
iaa.GaussianBlur(sigma=(1.0)),
iaa.ContrastNormalization((0.5, 1.5), per_channel=0.5)])
def activator_masks(images, augmenter, parents, default):
"""Activator for mask
activator for mask to control the augmentation of masks
DO NOT CHANGE THE INTERFACE OF THIS FUNCTION
"""
# Use augmentation on masks
if augmenter.name in ["Flipud", "Fliplr"]:
return default
# Do not use augmentation on masks
else:
return False
mask_hook = ia.HooksImages(activator=activator_masks)
return aug, mask_hook
def img_and_mask_augmentation(augmentation, img, mask):
"""
Augment image and bounding boxes !!
:param augmentation: augmentation settings
:param img: Only one image is needed. Not batch images
:param bbox: [[x1,y1,x2,y2],[x1,y1,x2,y2]...]
:return: Returns augment image and bbox
"""
# img_copy = img.copy()
image_shape = img.shape
# Convert the stochastic sequence of augmenters to a deterministic one.
# The deterministic sequence will always apply the exactly same effects to the images.
det = augmentation.to_deterministic()
img_aug = det.augment_image(img)
mask_aug = det.augment_image(mask,
hooks=imgaug.HooksImages(activator=hook))
mask_aug = mask_aug.astype(np.bool)
assert img_aug.shape == image_shape, "Augmentation shouldn't change image size"
return img_aug, mask_aug
def __getitem__(self, index):
annotation = self.annotations[index]
image = self.extract_image(annotation)
mask = self.extract_mask(annotation)
if self.augment:
det_tf = self.transform.to_deterministic()
image = det_tf.augment_image(image)
mask = det_tf.augment_image(
mask, hooks=ia.HooksImages(activator=self._activator_masks))
to_tensor = ToTensor()
image = to_tensor(image)
mask = to_tensor(mask)
# plt.imshow(mask[0,:,:])
# plt.show()
# opencvimage = np.hstack((cv2.normalize(image.astype('float'), None, 0.0, 1.0, cv2.NORM_MINMAX), cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)))
# cv2.imshow('Main', opencvimage)
# cv2.waitKey(10)
sample = {"image": image, "mask": mask}
return sample
def data_gen(x_train, y_train, bz, augmentation=None):
from sklearn.utils import shuffle
x_train, y_train = shuffle(x_train, y_train)
steps = len(x_train) // bz
n = 0
while True:
if n == steps:
n = 0
x_train, y_train = shuffle(x_train, y_train)
x, y = x_train[n * bz:(n + 1) * bz], y_train[n * bz:(n + 1) * bz]
n += 1
if augmentation:
import imgaug
# Augmentors that are safe to apply to masks
# Some, such as Affine, have settings that make them unsafe, so always
# test your augmentation on masks
MASK_AUGMENTERS = [
"Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr",
"Flipud", "CropAndPad", "Affine", "PiecewiseAffine"
]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return (augmenter.__class__.__name__ in MASK_AUGMENTERS)
# Make augmenters deterministic to apply similarly to images and masks
det = augmentation.to_deterministic()
x = det.augment_images(x)
# Change mask to np.uint8 because imgaug doesn't support np.bool
y = det.augment_images(y.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
x = (x / 127.5) - 1
yield x, y
