def test_determinism():
reseed()
images = [
ia.quokka(size=(128, 128)),
ia.quokka(size=(64, 64)),
ia.quokka((128, 256))
]
images.extend([ia.quokka(size=(16, 16))] * 20)
keypoints = [
ia.KeypointsOnImage([
ia.Keypoint(x=20, y=10), ia.Keypoint(x=5, y=5),
ia.Keypoint(x=10, y=43)], shape=(50, 60, 3))
] * 20
augs = [
iaa.Sequential([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.SomeOf(1, [iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.OneOf([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.Sometimes(0.5, iaa.Fliplr(1.0)),
iaa.WithColorspace("HSV", children=iaa.Add((-50, 50))),
iaa.Resize((0.5, 0.9)),
iaa.CropAndPad(px=(-50, 50)),
iaa.Pad(px=(1, 50)),
iaa.Crop(px=(1, 50)),
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Superpixels(p_replace=(0.25, 1.0), n_segments=(16, 128)),
iaa.Grayscale(alpha=(0.1, 1.0)),
iaa.GaussianBlur((0.1, 3.0)),
iaa.AverageBlur((3, 11)),
iaa.MedianBlur((3, 11)),
iaa.Sharpen(alpha=(0.1, 1.0), lightness=(0.8, 1.2)),
iaa.Emboss(alpha=(0.1, 1.0), strength=(0.8, 1.2)),
iaa.EdgeDetect(alpha=(0.1, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.1, 1.0), direction=(0.0, 1.0)),
iaa.Add((-50, 50)),
iaa.AddElementwise((-50, 50)),
iaa.AdditiveGaussianNoise(scale=(0.1, 1.0)),
iaa.Multiply((0.6, 1.4)),
iaa.MultiplyElementwise((0.6, 1.4)),
iaa.Dropout((0.3, 0.5)),
iaa.CoarseDropout((0.3, 0.5), size_percent=(0.05, 0.2)),
iaa.Invert(0.5),
iaa.Affine(scale=(0.7, 1.3), translate_percent=(-0.1, 0.1),
rotate=(-20, 20), shear=(-20, 20), order=ia.ALL,
mode=ia.ALL, cval=(0, 255)),
iaa.PiecewiseAffine(scale=(0.1, 0.3)),
iaa.ElasticTransformation(alpha=10.0)
]
augs_affect_geometry = [
iaa.Sequential([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.SomeOf(1, [iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.OneOf([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.Sometimes(0.5, iaa.Fliplr(1.0)),
iaa.Resize((0.5, 0.9)),
iaa.CropAndPad(px=(-50, 50)),
iaa.Pad(px=(1, 50)),
iaa.Crop(px=(1, 50)),
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Affine(scale=(0.7, 1.3), translate_percent=(-0.1, 0.1),
rotate=(-20, 20), shear=(-20, 20), order=ia.ALL,
mode=ia.ALL, cval=(0, 255)),
iaa.PiecewiseAffine(scale=(0.1, 0.3)),
iaa.ElasticTransformation(alpha=(5, 100), sigma=(3, 5))
]
for aug in augs:
aug_det = aug.to_deterministic()
images_aug1 = aug_det.augment_images(images)
images_aug2 = aug_det.augment_images(images)
aug_det = aug.to_deterministic()
images_aug3 = aug_det.augment_images(images)
images_aug4 = aug_det.augment_images(images)
assert array_equal_lists(images_aug1, images_aug2), \
"Images (1, 2) expected to be identical for %s" % (aug.name,)
assert array_equal_lists(images_aug3, images_aug4), \
"Images (3, 4) expected to be identical for %s" % (aug.name,)
assert not array_equal_lists(images_aug1, images_aug3), \
"Images (1, 3) expected to be different for %s" % (aug.name,)
for aug in augs_affect_geometry:
aug_det = aug.to_deterministic()
kps_aug1 = aug_det.augment_keypoints(keypoints)
kps_aug2 = aug_det.augment_keypoints(keypoints)
aug_det = aug.to_deterministic()
kps_aug3 = aug_det.augment_keypoints(keypoints)
kps_aug4 = aug_det.augment_keypoints(keypoints)
assert keypoints_equal(kps_aug1, kps_aug2), \
"Keypoints (1, 2) expected to be identical for %s" % (aug.name,)
assert keypoints_equal(kps_aug3, kps_aug4), \
"Keypoints (3, 4) expected to be identical for %s" % (aug.name,)
assert not keypoints_equal(kps_aug1, kps_aug3), \
"Keypoints (1, 3) expected to be different for %s" % (aug.name,)
def __init__(self):
super(ImgAugTransform, self).__init__()
from imgaug import augmenters as iaa
from imgaug import parameters as iap
self.seq = iaa.Sequential(children=[
iaa.Sequential(children=[
iaa.Sequential(children=[
iaa.OneOf(children=[
iaa.Sometimes(
p=0.95,
then_list=iaa.Affine(scale={
"x": (0.9, 1.1),
"y": (0.9, 1.1)
},
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-30, 30),
shear=(-15, 15),
order=iap.Choice(
[0, 1, 3],
p=[0.15, 0.80, 0.05]),
mode="reflect",
name="Affine")),
iaa.Sometimes(p=0.05,
then_list=iaa.PerspectiveTransform(
scale=(0.01, 0.1)))
],
name="Blur"),
iaa.Sometimes(p=0.01,
then_list=iaa.PiecewiseAffine(
scale=(0.0, 0.01),
nb_rows=(4, 20),
nb_cols=(4, 20),
order=iap.Choice([0, 1, 3],
p=[0.15, 0.80, 0.05]),
mode="reflect",
name="PiecewiseAffine"))
],
random_order=True,
name="GeomTransform"),
iaa.Sequential(children=[
iaa.Sometimes(p=0.75,
then_list=iaa.Add(value=(-10, 10),
per_channel=0.5,
name="Brightness")),
iaa.Sometimes(p=0.05,
then_list=iaa.Emboss(alpha=(0.0, 0.5),
strength=(0.5, 1.2),
name="Emboss")),
iaa.Sometimes(p=0.1,
then_list=iaa.Sharpen(alpha=(0.0, 0.5),
lightness=(0.5, 1.2),
name="Sharpen")),
iaa.Sometimes(p=0.25,
then_list=iaa.ContrastNormalization(
alpha=(0.5, 1.5),
per_channel=0.5,
name="ContrastNormalization"))
],
random_order=True,
name="ColorTransform"),
iaa.Sequential(children=[
iaa.Sometimes(p=0.5,
then_list=iaa.AdditiveGaussianNoise(
loc=0,
scale=(0.0, 10.0),
per_channel=0.5,
name="AdditiveGaussianNoise")),
iaa.Sometimes(p=0.1,
then_list=iaa.SaltAndPepper(
p=(0, 0.001),
per_channel=0.5,
name="SaltAndPepper"))
],
random_order=True,
name="Noise"),
iaa.OneOf(children=[
iaa.Sometimes(p=0.05,
then_list=iaa.MedianBlur(k=3,
name="MedianBlur")),
iaa.Sometimes(p=0.05,
then_list=iaa.AverageBlur(
k=(2, 4), name="AverageBlur")),
iaa.Sometimes(p=0.5,
then_list=iaa.GaussianBlur(
sigma=(0.0, 2.0), name="GaussianBlur"))
],
name="Blur"),
],
random_order=True,
name="MainProcess")
])
def get_kinetics(data_root=os.path.join('/media', 'user', 'disk0'),
name='KINETICS-700.2020',
clip_length=8,
clip_size=256,
val_clip_length=None,
val_clip_size=None,
train_interval=3,
val_interval=3,
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
seed=torch.distributed.get_rank()
if torch.distributed.is_initialized() else 0,
**kwargs):
""" data iter for kinetics
"""
logging.debug("VideoIter:: clip_length = {}, interval = [train: {}, val: {}], seed = {}".format( \
clip_length, train_interval, val_interval, seed))
sometimes_aug = lambda aug: iaa.Sometimes(0.4, aug)
sometimes_seq = lambda aug: iaa.Sometimes(0.9, aug)
train_sampler = sampler.RandomSampling(num=clip_length,
interval=train_interval,
speed=[1.0, 1.0],
seed=(seed + 0))
if ('200' in name):
extension = '200'
elif ('400' in name):
extension = '400'
elif ('600' in name):
extension = '600'
else:
extension = '700'
train = VideoIter(
dataset_location=os.path.join(data_root, 'data', 'Kinetics_videos',
'jpg'),
csv_filepath=os.path.join(data_root, 'labels',
'kinetics-' + extension + '_train' + '.csv'),
include_timeslices=True,
sampler=train_sampler,
video_size=(clip_length, clip_size, clip_size),
video_transform=transforms.Compose(transforms=iaa.Sequential([
iaa.Resize({
"shorter-side": 384,
"longer-side": "keep-aspect-ratio"
}),
iaa.CropToFixedSize(width=384, height=384, position='center'),
iaa.CropToFixedSize(width=clip_size,
height=clip_size,
position='uniform'),
sometimes_seq(
iaa.Sequential([
sometimes_aug(iaa.GaussianBlur(sigma=[0.1, 0.2, 0.3])),
sometimes_aug(iaa.Add((-5, 15), per_channel=True)),
sometimes_aug(iaa.AverageBlur(k=(1, 2))),
sometimes_aug(iaa.Multiply((0.8, 1.2))),
sometimes_aug(
iaa.GammaContrast((0.85, 1.15), per_channel=True)),
sometimes_aug(
iaa.AddToHueAndSaturation((-16, 16),
per_channel=True)),
sometimes_aug(iaa.LinearContrast((0.85, 1.115))),
sometimes_aug(
iaa.OneOf([
iaa.PerspectiveTransform(scale=(0.02, 0.05),
keep_size=True),
iaa.Rotate(rotate=(-10, 10)),
]))
])),
iaa.Fliplr(0.5)
]),
normalise=[mean, std]),
name='train',
shuffle_list_seed=(seed + 2))
# Only return train iterator
if (val_clip_length is None and val_clip_size is None):
return train
val_sampler = sampler.SequentialSampling(num=clip_length,
interval=val_interval,
fix_cursor=True,
shuffle=True)
val = VideoIter(
dataset_location=os.path.join(data_root, 'data', 'Kinetics_videos',
'jpg'),
csv_filepath=os.path.join(data_root, 'labels',
'kinetics-' + extension + '_val.csv'),
include_timeslices=True,
sampler=val_sampler,
video_size=(16, 256, 256),
video_transform=transforms.Compose(transforms=iaa.Sequential([
iaa.Resize({
"shorter-side": 294,
"longer-side": "keep-aspect-ratio"
}),
iaa.CropToFixedSize(width=294, height=294, position='center'),
iaa.CropToFixedSize(width=256, height=256, position='center')
]),
normalise=[mean, std]),
name='val')
return (train, val)
def _create_main_list(m, _cval):
return [iaa.Add(m)]
def __init__(self, images, config, shuffle=True, jitter=True, norm=None):
self.generator = None
self.images = images
self.config = config
self.shuffle = shuffle
self.jitter = jitter
self.norm = norm
self.anchors = [
BoundBox(0, 0, config['ANCHORS'][2 * i],
config['ANCHORS'][2 * i + 1])
for i in range(int(len(config['ANCHORS']) // 2))
]
### augmentors by https://github.com/aleju/imgaug
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
self.aug_pipe = iaa.Sequential(
[
# apply the following augmenters to most images
#iaa.Fliplr(0.5), # horizontally flip 50% of all images
#iaa.Flipud(0.2), # vertically flip 20% of all images
#sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
sometimes(
iaa.Affine(
#scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
#translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
#rotate=(-5, 5), # rotate by -45 to +45 degrees
#shear=(-5, 5), # shear by -16 to +16 degrees
#order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
#cval=(0, 255), # if mode is constant, use a cval between 0 and 255
#mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 5),
[
#sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0)
), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(
k=(2, 7)
), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(
k=(3, 11)
), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
#iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges
#sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
#])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255),
per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
#iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
#iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add(
(-10, 10), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.Multiply(
(0.5, 1.5), per_channel=0.5
), # change brightness of images (50-150% of original value)
iaa.ContrastNormalization(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
#iaa.Grayscale(alpha=(0.0, 1.0)),
#sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
#sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
],
random_order=True)
],
random_order=True)
if shuffle: np.random.shuffle(self.images)
iaa.Fliplr(0.5), # horizontally flip
iaa.OneOf([
iaa.Noop(),
iaa.Affine(rotate=(-6, 6),
translate_percent={"x": (-0.25, 0.25)},
mode='symmetric'),
iaa.Noop(),
iaa.CropAndPad(percent=(-0.2, 0.2),
pad_mode="reflect",
keep_size=False),
])
# More as you want ...
])
seq_train = iaa.Sequential(
sometimes(iaa.Multiply((0.8, 1.2))), # , per_channel=0.5
sometimes(iaa.Add((-0.2, 0.2))), # , per_channel=0.5
sometimes(
iaa.OneOf([
iaa.AdditiveGaussianNoise(scale=(0, 0.05)),
iaa.GaussianBlur(sigma=(0.0, 1.0)),
])))
def make_image_gen(features, labels, batch_size=32):
num = features.shape[0]
all_batches_index = np.arange(0, num)
while True:
np.random.shuffle(all_batches_index)
for i in range(0, num, batch_size):
out_images, out_masks = [], []
for index in all_batches_index[i:i + batch_size]:
iaa.MedianBlur(
k=(1, 5)
), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.03 * 255),
per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.01, 0.03),
per_channel=0.2),
]),
iaa.Add(
(-10, 10), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.ContrastNormalization(
(0.3, 1.0),
per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(
0.01, 0.05))), # sometimes move parts of the image around
],
random_order=True)
],
random_order=True)
mode='symmetric'),
]),
# Deformations
iaa.Sometimes(0.3, iaa.PiecewiseAffine(scale=(0.04, 0.08))),
iaa.Sometimes(0.3, iaa.PerspectiveTransform(scale=(0.05, 0.1))),
],
random_order=True)
intensity_seq = iaa.Sequential([
iaa.Invert(0.3),
iaa.Sometimes(0.3, iaa.ContrastNormalization((0.5, 1.5))),
iaa.OneOf([
iaa.Noop(),
iaa.Sequential([
iaa.OneOf([
iaa.Add((-10, 10)),
iaa.AddElementwise((-10, 10)),
iaa.Multiply((0.95, 1.05)),
iaa.MultiplyElementwise((0.95, 1.05)),
]),
]),
iaa.OneOf([
iaa.GaussianBlur(sigma=(0.0, 1.0)),
iaa.AverageBlur(k=(2, 5)),
iaa.MedianBlur(k=(3, 5))
])
])
],
random_order=False)
def _get_example(self, files_dir, idx):
if osp.exists(osp.join(files_dir, 'rgb_obj_y.jpg')):
image_file = osp.join(files_dir, 'rgb_obj_y.jpg')
elif osp.exists(osp.join(files_dir, 'rgb_obj_g2.jpg')):
image_file = osp.join(files_dir, 'rgb_obj_g2.jpg')
else:
image_file = osp.join(files_dir, 'rgb_obj_g.jpg')
image = skimage.io.imread(image_file)
assert image.dtype == np.uint8
assert image.ndim == 3
if osp.exists(osp.join(files_dir, 'depth_obj_y.npz')):
depth_file = osp.join(files_dir, 'depth_obj_y.npz')
else:
depth_file = osp.join(files_dir, 'depth_obj_n.npz')
depth = np.load(depth_file)['arr_0']
depth[depth == 0] = np.nan
if depth.dtype == np.uint16:
depth = depth.astype(np.float32)
depth /= 1000
depth_keep = ~np.isnan(depth)
depth[depth_keep] = np.maximum(depth[depth_keep], self.min_value)
depth[depth_keep] = np.minimum(depth[depth_keep], self.max_value)
assert depth.dtype == np.float32
assert depth.ndim == 2
label_file = osp.join(files_dir, 'mask_obj_g.jpg')
with open(label_file, 'r') as f:
label = np.asarray(PIL.Image.open(f)).astype(np.int32)
assert label.dtype == np.int32
assert label.ndim == 2
depth_gt_file = osp.join(files_dir, 'depth_obj_g.npz')
depth_gt = np.load(depth_gt_file)['arr_0']
depth_gt[depth_gt == 0] = np.nan
if depth_gt.dtype == np.uint16:
depth_gt = depth_gt.astype(np.float32)
depth /= 1000
depth_gt_keep = ~np.isnan(depth_gt)
depth_gt[depth_gt_keep] = np.maximum(depth_gt[depth_gt_keep],
self.min_value)
depth_gt[depth_gt_keep] = np.minimum(depth_gt[depth_gt_keep],
self.max_value)
assert depth_gt.dtype == np.float32
assert depth_gt.ndim == 2
# Data augmentation
if self.aug:
# 1. Color augmentation
obj_datum = dict(img=image)
random_state = np.random.RandomState()
def st(x):
return iaa.Sometimes(0.3, x)
augs = [
st(iaa.Add([-50, 50], per_channel=True)),
st(
iaa.InColorspace('HSV',
children=iaa.WithChannels(
[1, 2], iaa.Multiply([0.5, 2])))),
st(iaa.GaussianBlur(sigma=[0.0, 1.0])),
st(
iaa.AdditiveGaussianNoise(scale=(0.0, 0.1 * 255),
per_channel=True)),
]
obj_datum = next(
mvtk.aug.augment_object_data([obj_datum],
random_state=random_state,
augmentations=augs))
image = obj_datum['img']
# 2. Depth noise
np.random.seed()
if np.random.uniform() < 0.3:
noise_rate = np.random.uniform() * 0.25 + 0.05
depth[np.random.rand(depth.shape[0], depth.shape[1]) <
noise_rate] = np.nan
# 3. Geometric augmentation
if np.random.uniform() < 0.5:
image = np.fliplr(image)
depth = np.fliplr(depth)
label = np.fliplr(label)
depth_gt = np.fliplr(depth_gt)
if np.random.uniform() < 0.5:
image = np.flipud(image)
depth = np.flipud(depth)
label = np.flipud(label)
depth_gt = np.flipud(depth_gt)
if np.random.uniform() < 0.5:
angle = (np.random.uniform() * 180) - 90
image = self.rotate_image(image, angle, cv2.INTER_LINEAR)
depth = self.rotate_depth_image(depth, angle, cv2.INTER_LINEAR)
label = self.rotate_image(label, angle, cv2.INTER_NEAREST)
depth_gt = self.rotate_depth_image(depth_gt, angle,
cv2.INTER_LINEAR)
return image, depth, label, depth_gt, idx
def __init__(self,
mask_dir=None,
root=None,
mask_type=None,
transforms=None,
is_train=True):
print('Dataset LiverBgAugpolygonDataset')
print(
'Dataset Init: \n\tmask_dir={}, \n\troot={}, \n\tmask_type={}, \n\ttransforms={}, \n\tis_train={}'
.format(mask_dir, root, mask_type, transforms, is_train))
# "mask_type" = "polygon" or "image"
# for debug
print('data loader init args')
init_list = [mask_dir, root, mask_type, transforms, is_train]
print('\t' +
'\n\t'.join([str(arg) if arg else 'None' for arg in init_list]))
# end
# norm path
root = norm_path(root)
mask_dir = norm_path(mask_dir)
self.mask_type = 'polygon' # mask_type
self.transforms = transforms
self.image_size = 512
self.is_train = is_train
self.img_key_list = list()
self.img_dict = dict()
self.ann_info = dict()
cls = LiverBgAugpolygonDataset.CLASSES
self.class_to_ind = dict(zip(cls, range(len(cls))))
self.img_dict = image_dict(root,
exts=['.png', '.jpg', '.jpeg'],
recursive=True,
followlinks=True)
for cls_num, cls_name in enumerate(self.CLASSES[1:], 1):
cls_mask_path = os.path.join(mask_dir, cls_name)
assert os.path.exists(
cls_mask_path), "Not found class({}) path: {}".format(
cls, cls_mask_path)
mask_dict = image_dict(cls_mask_path)
for mask_key, mask_file in mask_dict.items():
if mask_key not in self.ann_info:
self.ann_info[mask_key] = list()
# x, y, w, h = None, None, None, None
# self.ann_info[mask_key].append([x, y, w, h, cls_num, mask_file])
self.ann_info[mask_key].append([cls_num, mask_file])
self.img_key_list = list(set(self.img_dict) & set(self.ann_info))
print('found images', len(self.img_dict))
print('found masks', len(self.ann_info))
print('using image&mask', len(self.img_key_list))
# self.train_augmentation = iaa.Sequential([
# # iaa.PiecewiseAffine(scale=(0.00, 0.05), nb_cols=3, nb_rows=3),
# # iaa.Affine(rotate=(-20, 20)),
# iaa.SomeOf((0, None), [
# # iaa.Fliplr(0.5),
# iaa.Multiply((0.5, 1.5)),
# iaa.Add((-10, 10)),
# iaa.GaussianBlur(sigma=(0, 1.0))
# ], random_order=False)
# ])
if self.is_train:
self.augmentation = iaa.Sequential([
iaa.SomeOf(
(0, None),
[
iaa.Fliplr(0.5),
# iaa.PiecewiseAffine(scale=(0.00, 0.05), nb_cols=3, nb_rows=3),
iaa.Affine(rotate=(-20, 20)),
iaa.Affine(translate_px={
"x": (-50, 50),
"y": (-50, 50)
}),
iaa.Affine(scale=(0.5, 1.5)),
iaa.Multiply((0.5, 1.5)),
iaa.Add((-10, 10)),
iaa.GaussianBlur(sigma=(0, 1.0))
],
random_order=False)
])
else:
self.augmentation = iaa.Sequential([], random_order=False)
def augument(self,image):
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
augment_img = iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
sometimes(iaa.Affine(
scale={"x": (0.9, 1.1), "y": (0.9, 1.1)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.1, 0.1), "y": (-0.1, 0.1)}, # translate by -20 to +20 percent (per axis)
rotate=(-10, 10), # rotate by -45 to +45 degrees
shear=(-5, 5), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 1.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(3, 5)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)), # sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.01*255), per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.05), per_channel=0.5), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.01, 0.03), size_percent=(0.01, 0.02), per_channel=0.2),
]),
iaa.Invert(0.01, per_channel=True), # invert color channels
iaa.Add((-2, 2), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-1, 1)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.9, 1.1), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-1, 0),
first=iaa.Multiply((0.9, 1.1), per_channel=True),
second=iaa.ContrastNormalization((0.9, 1.1))
)
]),
sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True
)
],
random_order=True
)
image_aug = augment_img.augment_image(image)
return image_aug
def __init__(self):
self.imgaug_transform = iaa.Add((100, 100), per_channel=False)
def __init__(self):
self.imgaug_transform = iaa.Add((127, 127), per_channel=False)
self.augmentor_op = Operations.RandomBrightness(probability=1, min_factor=1.5, max_factor=1.5)
self.solt_stream = slc.Stream([slt.ImageRandomBrightness(p=1, brightness_range=(127, 127))])
def test_dtype_preservation():
reseed()
size = (4, 16, 16, 3)
images = [
np.random.uniform(0, 255, size).astype(np.uint8),
np.random.uniform(0, 65535, size).astype(np.uint16),
np.random.uniform(0, 4294967295, size).astype(np.uint32),
np.random.uniform(-128, 127, size).astype(np.int16),
np.random.uniform(-32768, 32767, size).astype(np.int32),
np.random.uniform(0.0, 1.0, size).astype(np.float32),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float16),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float32),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float64)
]
default_dtypes = set([arr.dtype for arr in images])
# Some dtypes are here removed per augmenter, because the respective
# augmenter does not support them. This test currently only checks whether
# dtypes are preserved from in- to output for all dtypes that are supported
# per augmenter.
# dtypes are here removed via list comprehension instead of
# `default_dtypes - set([dtype])`, because the latter one simply never
# removed the dtype(s) for some reason
def _not_dts(dts):
return [dt for dt in default_dtypes if dt not in dts]
augs = [
(iaa.Add((-5, 5), name="Add"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.AddElementwise((-5, 5), name="AddElementwise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.AdditiveGaussianNoise(0.01*255, name="AdditiveGaussianNoise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Multiply((0.95, 1.05), name="Multiply"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Dropout(0.01, name="Dropout"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.CoarseDropout(0.01, size_px=6, name="CoarseDropout"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Invert(0.01, per_channel=True, name="Invert"),
default_dtypes),
(iaa.GaussianBlur(sigma=(0.95, 1.05), name="GaussianBlur"),
_not_dts([np.float16])),
(iaa.AverageBlur((3, 5), name="AverageBlur"),
_not_dts([np.uint32, np.int32, np.float16])),
(iaa.MedianBlur((3, 5), name="MedianBlur"),
_not_dts([np.uint32, np.int32, np.float16, np.float64])),
(iaa.BilateralBlur((3, 5), name="BilateralBlur"),
_not_dts([np.uint16, np.uint32, np.int16, np.int32, np.float16,
np.float64])),
(iaa.Sharpen((0.0, 0.1), lightness=(1.0, 1.2), name="Sharpen"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.Emboss(alpha=(0.0, 0.1), strength=(0.5, 1.5), name="Emboss"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.EdgeDetect(alpha=(0.0, 0.1), name="EdgeDetect"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.DirectedEdgeDetect(alpha=(0.0, 0.1), direction=0,
name="DirectedEdgeDetect"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.Fliplr(0.5, name="Fliplr"), default_dtypes),
(iaa.Flipud(0.5, name="Flipud"), default_dtypes),
(iaa.Affine(translate_px=(-5, 5), name="Affine-translate-px"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(translate_percent=(-0.05, 0.05),
name="Affine-translate-percent"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(rotate=(-20, 20), name="Affine-rotate"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(shear=(-20, 20), name="Affine-shear"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(scale=(0.9, 1.1), name="Affine-scale"),
_not_dts([np.uint32, np.int32])),
(iaa.PiecewiseAffine(scale=(0.001, 0.005), name="PiecewiseAffine"),
default_dtypes),
(iaa.ElasticTransformation(alpha=(0.1, 0.2), sigma=(0.1, 0.2),
name="ElasticTransformation"),
_not_dts([np.float16])),
(iaa.Sequential([iaa.Identity(), iaa.Identity()],
name="SequentialNoop"),
default_dtypes),
(iaa.SomeOf(1, [iaa.Identity(), iaa.Identity()], name="SomeOfNoop"),
default_dtypes),
(iaa.OneOf([iaa.Identity(), iaa.Identity()], name="OneOfNoop"),
default_dtypes),
(iaa.Sometimes(0.5, iaa.Identity(), name="SometimesNoop"),
default_dtypes),
(iaa.Sequential([iaa.Add((-5, 5)), iaa.AddElementwise((-5, 5))],
name="Sequential"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.SomeOf(1,
[iaa.Add((-5, 5)), iaa.AddElementwise((-5, 5))],
name="SomeOf"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.OneOf([iaa.Add((-5, 5)), iaa.AddElementwise((-5, 5))],
name="OneOf"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Sometimes(0.5, iaa.Add((-5, 5)), name="Sometimes"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Identity(name="Identity"), default_dtypes),
(iaa.BlendAlpha((0.0, 0.1), iaa.Identity(), name="BlendAlphaIdentity"),
default_dtypes),
(iaa.BlendAlphaElementwise((0.0, 0.1), iaa.Identity(),
name="BlendAlphaElementwiseIdentity"),
default_dtypes),
(iaa.BlendAlphaSimplexNoise(iaa.Identity(),
name="BlendAlphaSimplexNoiseIdentity"),
default_dtypes),
(iaa.BlendAlphaFrequencyNoise(exponent=(-2, 2),
foreground=iaa.Identity(),
name="BlendAlphaFrequencyNoiseIdentity"),
default_dtypes),
(iaa.BlendAlpha((0.0, 0.1), iaa.Add(10), name="BlendAlpha"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.BlendAlphaElementwise((0.0, 0.1), iaa.Add(10),
name="BlendAlphaElementwise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.BlendAlphaSimplexNoise(iaa.Add(10), name="BlendAlphaSimplexNoise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.BlendAlphaFrequencyNoise(exponent=(-2, 2),
foreground=iaa.Add(10),
name="BlendAlphaFrequencyNoise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Superpixels(p_replace=0.01, n_segments=64),
_not_dts([np.float16, np.float32, np.float64])),
(iaa.Resize({"height": 4, "width": 4}, name="Resize"),
_not_dts([np.uint16, np.uint32, np.int16, np.int32, np.float32,
np.float16, np.float64])),
(iaa.CropAndPad(px=(-10, 10), name="CropAndPad"),
_not_dts([np.uint16, np.uint32, np.int16, np.int32, np.float32,
np.float16, np.float64])),
(iaa.Pad(px=(0, 10), name="Pad"),
_not_dts([np.uint16, np.uint32, np.int16, np.int32, np.float32,
np.float16, np.float64])),
(iaa.Crop(px=(0, 10), name="Crop"),
_not_dts([np.uint16, np.uint32, np.int16, np.int32, np.float32,
np.float16, np.float64]))
]
for (aug, allowed_dtypes) in augs:
for images_i in images:
if images_i.dtype in allowed_dtypes:
images_aug = aug.augment_images(images_i)
assert images_aug.dtype == images_i.dtype
def get_augmenter(name, c_val=255, vertical_flip=True):
if name:
alot = lambda aug: iaa.Sometimes(0.75, aug)
alw = lambda aug: iaa.Sometimes(1, aug)
sometimes = lambda aug: iaa.Sometimes(0.50, aug)
few = lambda aug: iaa.Sometimes(0.10, aug)
if 'rgb' in name:
scale = random.uniform(0.87, 1.25)
seq_rgb = iaa.Sequential([
iaa.Fliplr(0.50), # horizontally flip 50% of the images
iaa.Flipud(0.20), # vertically flip 50% of the images
sometimes(iaa.Add((-30, 30))),
sometimes(iaa.Multiply((0.80, 1.20), per_channel=False)),
sometimes(iaa.GaussianBlur(sigma=(0, 0.10))),
few(iaa.CoarseDropout(p=(0.05, 0.15), size_percent=(0.15, 0.35), per_channel=True)),
few(iaa.CoarseDropout(p=(0.05, 0.15), size_percent=(0.15, 0.35), per_channel=False)),
sometimes(iaa.ContrastNormalization((0.75, 1.35))),
alot(iaa.Affine(
scale={"x": (scale), "y": (scale)},
# scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
# translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
order=1, #bilinear interpolation (fast)
cval=0,
mode="reflect" # `edge`, `wrap`, `reflect` or `symmetric`
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
))])
return seq_rgb
elif 'segmentation' in name:
#create one per image. give iamge, label and mask to the pipeling
value_flip = round(random.random())
if value_flip>0.5:
value_flip=1
else:
value_flip=0
value_flip2 = round(random.random())
if value_flip2>0.5:
value_flip2=1
else:
value_flip2=0
value_add = random.uniform(-10, 10)
value_Multiply = random.uniform(0.95, 1.10)
#value_GaussianBlur = random.uniform(0.0,0.05)
ContrastNormalization = random.uniform(0.90, 1.20)
scale = random.uniform(0.50, 2)
value_x2 = random.uniform(-0.20, 0.20)
value_y2 = random.uniform(-0.20, 0.20)
val_rotate = random.uniform(-10,10)
seq_image = iaa.Sequential([
sometimes(iaa.Add((value_add, value_add))),
sometimes(iaa.Multiply((value_Multiply, value_Multiply), per_channel=False)),
#sometimes(iaa.GaussianBlur(sigma=(value_GaussianBlur, value_GaussianBlur))),
sometimes(iaa.ContrastNormalization((ContrastNormalization, ContrastNormalization))),
iaa.Fliplr(value_flip), # horizontally flip 50% of the images
#iaa.Flipud(value_flip2), # vertically flip 50% of the images
iaa.Affine(
scale={"x": (scale), "y": (scale)},
# scale images to 80-120% of their size, individually per axis
translate_percent={"x": (value_x2), "y": (value_y2)},
# translate by -20 to +20 percent (per axis)
rotate=(val_rotate), # rotate by -45 to +45 degrees
order=1, #bilinear interpolation (fast)
cval=c_val,
mode="reflect",
# `edge`, `wrap`, `reflect` or `symmetric`
# cval=c_val, # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)])
seq_label = iaa.Sequential([
iaa.Fliplr(value_flip), # horizontally flip 50% of the images
#iaa.Flipud(value_flip2), # vertically flip 50% of the images
iaa.Affine(
scale={"x": (scale), "y": (scale)},
# scale images to 80-120% of their size, individually per axis
translate_percent={"x": (value_x2), "y": (value_y2)},
# translate by -20 to +20 percent (per axis)
rotate=(val_rotate), # rotate by -45 to +45 degrees
order=0, #bilinear interpolation (fast)
cval=c_val,
mode="constant" # `edge`, `wrap`, `reflect` or `symmetric`
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)])
seq_events = iaa.Sequential([
iaa.Fliplr(value_flip), # horizontally flip 50% of the images
#iaa.Flipud(value_flip2), # vertically flip 50% of the images
iaa.Affine(
scale={"x": (scale), "y": (scale)},
# scale images to 80-120% of their size, individually per axis
translate_percent={"x": (value_x2), "y": (value_y2)},
# translate by -20 to +20 percent (per axis)
rotate=(val_rotate), # rotate by -45 to +45 degrees
order=0, #bilinear interpolation (fast)
cval=0,
mode="constant" # `edge`, `wrap`, `reflect` or `symmetric`
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)])
seq_mask = iaa.Sequential([
iaa.Fliplr(value_flip), # horizontally flip 50% of the images
#iaa.Flipud(value_flip2), # vertically flip 50% of the images
iaa.Affine(
scale={"x": (scale), "y": (scale)},
# scale images to 80-120% of their size, individually per axis
translate_percent={"x": (value_x2), "y": (value_y2)},
# translate by -20 to +20 percent (per axis)
rotate=(val_rotate), # rotate by -45 to +45 degrees
order=0, #bilinear interpolation (fast)
cval=0,
mode="constant" # `edge`, `wrap`, `reflect` or `symmetric`
# cval=c_val, # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)])
return seq_image, seq_label, seq_mask, seq_events
else:
seq_multi = iaa.Sequential([
sometimes(iaa.Affine(
# scale images to 80-120% of their size, individually per axis
# translate by -20 to +20 percent (per axis)
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
# scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.20, 0.2), "y": (-0.2, 0.2)},
# translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
order=0, # use nearest neighbour
cval=127.5,
mode="constant"
))
])
return seq_multi
else:
return None
def __init__(self,
config,
dataset,
num_episodes,
shuffle=True,
jitter=True):
'Initialization'
self.config = config
self.dataset = dataset
self.batch_size = self.config['train']['batch_size']
self.num_episodes = num_episodes
self.images_per_episode = config['lstm']['images_per_episode']
self.batches_per_episode = int(self.images_per_episode /
self.batch_size)
self.current_episode = 0
self.step_counter = 0
self.batch_counter = 0
self.image_h = config['models']['road_seg_module']['image-size'][
'height']
self.image_w = config['models']['road_seg_module']['image-size'][
'width']
self.image_channels = config['models']['road_seg_module'][
'image-channels']
self.shuffle = shuffle
self.jitter = jitter
self.on_epoch_end()
st = lambda aug: iaa.Sometimes(0.4, aug)
oc = lambda aug: iaa.Sometimes(0.3, aug)
rl = lambda aug: iaa.Sometimes(0.09, aug)
self.augment = iaa.Sequential(
[
rl(iaa.GaussianBlur(
(0, 1.5))), # blur images with a sigma between 0 and 1.5
rl(
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05),
per_channel=0.5)), # add gaussian noise to images
oc(iaa.Dropout((0.0, 0.10), per_channel=0.5)
), # randomly remove up to X% of the pixels
oc(
iaa.CoarseDropout(
(0.0, 0.10), size_percent=(0.08, 0.2),
per_channel=0.5)),
# randomly remove up to X% of the pixels
oc(
iaa.Add((-40, 40), per_channel=0.5)
), # change brightness of images (by -X to Y of original value)
st(iaa.Multiply((0.10, 2.5), per_channel=0.2)
), # change brightness of images (X-Y% of original value)
rl(iaa.ContrastNormalization(
(0.5, 1.5),
per_channel=0.5)), # improve or worsen the contrast
rl(iaa.Grayscale((0.0, 1))), # put grayscale
],
random_order=True # do all of the above in random order
)
class Augmentator:
Sequences = {
# ============================================
'aug1': iaa.Sequential([
iaa.Grayscale(alpha=(1.0, 1.0)),
do(1., iaa.Affine(
scale = {"x": (+0.8, +1.0), "y": (+0.8, +1.0)},
translate_percent = {"x": (-0.1, +0.1), "y": (-0.1, +0.1)},
rotate = (-45, +45),
shear = ( -5, +5),
order = [0, 1],
cval = (220, 255),
)),
do(.5, iaa.AdditiveGaussianNoise(scale=(0, 0.1*255))),
do(.5, iaa.Multiply((0.5, 1.2))),
do(.3, iaa.GaussianBlur(sigma=(0, 1.0))),
do(.5, iaa.ContrastNormalization((0.5, 1.5))),
do(.5, iaa.FrequencyNoiseAlpha(
exponent = (-4, -1),
first = iaa.Multiply(iaa.Choice((0.5, 2.0)), per_channel=False),
size_px_max = 32,
upscale_method = "linear",
iterations = 1,
sigmoid = True)),
]),
# ============================================
# ============================================
'aug2': iaa.Sequential([
iaa.Grayscale(alpha=(1.0, 1.0)),
do(1., iaa.Affine(
scale = {"x": (+0.95, +1.0), "y": (+0.95, +1.0)},
translate_percent = {"x": (-0.05, +0.05), "y": (-0.05, +0.05)},
rotate = (-15, +15),
shear = ( -1, +1),
order = [0, 1],
cval = (230, 255),
)),
do(.5, iaa.GaussianBlur(sigma=(.0,.5))),
do(.5, iaa.AdditiveGaussianNoise(scale=(0, 10))),
do(1., iaa.Add((-30,+30))),
do(1., iaa.Multiply((.95, 1.05))),
do(1., iaa.ContrastNormalization((0.75, 1.25))),
do(.2, iaa.FrequencyNoiseAlpha(
exponent = -4,
first = iaa.Invert(1, per_channel=False),
size_px_max = 64,
upscale_method = "linear",
iterations = 1,
sigmoid = True)),
do(.5, iaa.FrequencyNoiseAlpha(
exponent = (-5, -2),
first = iaa.Multiply(iaa.Choice((0.7, 1.1)), per_channel=False),
size_px_max = 32,
upscale_method = "linear",
iterations = 1,
sigmoid = True)),
]),
# ============================================
}
def __init__(self, name, seed=None):
ia.seed(seed)
self.name = name
self._seq = self.Sequences[name]
def augment(self, images):
return self._seq.augment_images(images)
model.load_weights(model_path, model_path, by_name=True)
# Train or evaluate
if args.command == "train":
if args.aug:
if not config.USE_OBJECT_ZOOM:
# Image Augmentation Pipeline
augmentation_pipeline = iaa.Sequential(
[
iaa.AdditiveGaussianNoise(scale=0.01 * 255,
name="AWGN"),
iaa.GaussianBlur(sigma=(0.0, 3.0), name="Blur"),
# iaa.Dropout([0.0, 0.05], name='Dropout'), # drop 0-5% of all pixels
iaa.Fliplr(0.5),
iaa.Add((-20, 20), name="Add"),
iaa.Multiply((0.8, 1.2), name="Multiply"),
iaa.Affine(scale=(0.8, 2.0)),
iaa.Affine(translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
}),
iaa.Affine(rotate=(-45,
45)), # rotate by -45 to 45 degrees
],
random_order=True)
else:
# Nevermind the image translation and scaling as this is done already during zoom in
augmentation_pipeline = iaa.Sequential(
[
iaa.AdditiveGaussianNoise(scale=0.01 * 255,
])
aug2 = iaa.SomeOf((2,7), [
iaa.Crop(px=(0,7)),
iaa.SaltAndPepper(p=(0,0.05)),
iaa.Dropout(p=(0, 0.1)),
iaa.SomeOf((1,3),[
iaa.GaussianBlur(sigma=(1.2, 2)),
iaa.AverageBlur(k=(1, 3)),
iaa.MotionBlur(angle=(36, 108), k=(3,6))
]),
iaa.AdditiveGaussianNoise(scale=(0, 0.01*255)),
# iaa.Sharpen(alpha=(0.0, 1.0), lightness=(0.2, 1.0)),
# iaa.Emboss(alpha=(0.0, 1.0), strength=(0.5, 1.0)),
iaa.Add((-30, 30)),
iaa.AddElementwise((-10, 10)),
iaa.MultiplyElementwise((.95, 1.05)),
iaa.ContrastNormalization((0.8, 1.2)),
iaa.Affine(shear=(-5, 5)),
iaa.Affine(rotate=(-4, 4))
], random_order=False)
# aug1 = iaa.SomeOf((0, 1), [
# iaa.Grayscale(alpha=(0, 1.0)),
# iaa.GammaContrast(per_channel=True, gamma=(0, 1.75))
# ])
# aug2 = iaa.SomeOf((2,7), [
# iaa.Crop(px=(0,5)),
# iaa.CoarsePepper(p=(0,0.1), size_percent=(0.6, 0.3)),
def augment_images(annotation_file,data_path,augment_times):
"""
This function will create augmented images and the corresponding masks as .jpg files
"""
print('This may take some time, as we are working with the original image size...')
annotation_file=data_path+annotation_file
#load coco anotation
coco = COCO(annotation_file)
#get images ids
images_id_array=coco.getImgIds()
#images_id_array=[2,3] #for testing
for image_id in images_id_array:
# Load image and mask with categories
image_ori = util.load_image(coco,image_id,data_path)
masks = util.load_image_masks(coco,image_id,image_ori.shape)
#folder to save images and mask
mask_dir=data_path+"batch_aug_ann/"
img_dir=data_path+"batch_aug/"
#number of augmentations can depend on the category
if augment_times is not None:
#get categories on image
cats_on_image=get_cat_on_image(coco,image_id)
#get maximum number to augment based on the categories in the image
nr_augmentations=0
for category in cats_on_image:
if category in augment_times.keys(): #if we have the number to augment
times=augment_times[category]
if times>nr_augmentations:
nr_augmentations=times
else:
nr_augmentations = 1
image = image_ori
# Define our augmentation pipeline.
seq = iaa.Sequential([
iaa.AdditiveGaussianNoise(scale=0.01 * 255, name="AWGN"),
#iaa.Affine(rotate=(-15, 15)), # rotate by -45 to 45 degrees
iaa.GaussianBlur(sigma=(0.0, 3.0), name="Blur"),
iaa.Dropout([0.0, 0.05], name='Dropout'), # drop 0-5% of all pixels
#iaa.Fliplr(0.5),
iaa.Add((-20, 20),name="Add"),
iaa.Multiply((0.8, 1.2), name="Multiply"),
iaa.Affine(scale=(0.8, 1.1)),
iaa.Affine(translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}),
iaa.Affine(rotate=(-45, 45)), # rotate by -45 to 45 degrees
], random_order=True)
print('image_id:'+str(image_id)+"num augmentations: "+str(nr_augmentations))
# Augment images and masks
for i in range(nr_augmentations):
image_aug=None
segmentations_aug=[]
seq_det = seq.to_deterministic()
image_aug_name=img_dir+str(image_id)+"_"+str(i)+".jpg"
#check all masks
for j in range(len(masks)):
segmap=masks[j]["segmentation"]
image_aug, segmap_aug = seq_det.augment(image=image, segmentation_maps=segmap)
segmentations_aug.append(segmap_aug)
category=masks[j]["category"]
mask_name=str(mask_dir)+str(image_id)+"_"+str(i)+"_"+category+"_"+str(j)+".png"
imageio.imwrite(mask_name, segmap_aug.draw()[0]) #save mask as image
#save image to folder
imageio.imwrite(image_aug_name, image_aug)
#plot image
#ia.imshow(np.hstack([ x.draw_on_image(image_aug)[0] for x in segmentations_aug
#]))
print("I finished procesign images!")
augmenters.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
sometimes(
augmenters.OneOf([
augmenters.EdgeDetect(alpha=(0, 0.7)),
augmenters.DirectedEdgeDetect(alpha=(0, 0.7),
direction=(0.0, 1.0)),
])),
augmenters.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
augmenters.OneOf([
augmenters.Dropout((0.01, 0.1), per_channel=0.5),
augmenters.CoarseDropout(
(0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
augmenters.Invert(0.05, per_channel=True),
augmenters.Add((-10, 10), per_channel=0.5),
augmenters.Multiply((0.5, 1.5), per_channel=0.5),
augmenters.ContrastNormalization((0.5, 2.0), per_channel=0.5),
augmenters.Grayscale(alpha=(0.0, 1.0)),
sometimes(
augmenters.ElasticTransformation(alpha=(0.5, 3.5),
sigma=0.25)),
sometimes(augmenters.PiecewiseAffine(scale=(0.01, 0.05)))
],
random_order=True)
aug_file_suffix = '_aug_{}'.format(i)
try:
augmented_image_df = augment_image(labels_df, DATASET_IMAGES_DIR,
DATASET_AUGM_IMAGES_DIR,
aug_file_suffix, aug_config)
image_annotation_dest_path = "{}/{}{}{}".format(
def create_augmenters(height, width, height_augmentable, width_augmentable,
only_augmenters):
def lambda_func_images(images, random_state, parents, hooks):
return images
def lambda_func_heatmaps(heatmaps, random_state, parents, hooks):
return heatmaps
def lambda_func_keypoints(keypoints, random_state, parents, hooks):
return keypoints
def assertlambda_func_images(images, random_state, parents, hooks):
return True
def assertlambda_func_heatmaps(heatmaps, random_state, parents, hooks):
return True
def assertlambda_func_keypoints(keypoints, random_state, parents, hooks):
return True
augmenters_meta = [
iaa.Sequential([iaa.Noop(), iaa.Noop()],
random_order=False,
name="Sequential_2xNoop"),
iaa.Sequential([iaa.Noop(), iaa.Noop()],
random_order=True,
name="Sequential_2xNoop_random_order"),
iaa.SomeOf((1, 3),
[iaa.Noop(), iaa.Noop(), iaa.Noop()],
random_order=False,
name="SomeOf_3xNoop"),
iaa.SomeOf((1, 3),
[iaa.Noop(), iaa.Noop(), iaa.Noop()],
random_order=True,
name="SomeOf_3xNoop_random_order"),
iaa.OneOf([iaa.Noop(), iaa.Noop(), iaa.Noop()], name="OneOf_3xNoop"),
iaa.Sometimes(0.5, iaa.Noop(), name="Sometimes_Noop"),
iaa.WithChannels([1, 2], iaa.Noop(), name="WithChannels_1_and_2_Noop"),
iaa.Noop(name="Noop"),
iaa.Lambda(func_images=lambda_func_images,
func_heatmaps=lambda_func_heatmaps,
func_keypoints=lambda_func_keypoints,
name="Lambda"),
iaa.AssertLambda(func_images=assertlambda_func_images,
func_heatmaps=assertlambda_func_heatmaps,
func_keypoints=assertlambda_func_keypoints,
name="AssertLambda"),
iaa.AssertShape((None, height_augmentable, width_augmentable, None),
name="AssertShape"),
iaa.ChannelShuffle(0.5, name="ChannelShuffle")
]
augmenters_arithmetic = [
iaa.Add((-10, 10), name="Add"),
iaa.AddElementwise((-10, 10), name="AddElementwise"),
#iaa.AddElementwise((-500, 500), name="AddElementwise"),
iaa.AdditiveGaussianNoise(scale=(5, 10), name="AdditiveGaussianNoise"),
iaa.AdditiveLaplaceNoise(scale=(5, 10), name="AdditiveLaplaceNoise"),
iaa.AdditivePoissonNoise(lam=(1, 5), name="AdditivePoissonNoise"),
iaa.Multiply((0.5, 1.5), name="Multiply"),
iaa.MultiplyElementwise((0.5, 1.5), name="MultiplyElementwise"),
iaa.Dropout((0.01, 0.05), name="Dropout"),
iaa.CoarseDropout((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarseDropout"),
iaa.ReplaceElementwise((0.01, 0.05), (0, 255),
name="ReplaceElementwise"),
#iaa.ReplaceElementwise((0.95, 0.99), (0, 255), name="ReplaceElementwise"),
iaa.ImpulseNoise((0.01, 0.05), name="ImpulseNoise"),
iaa.SaltAndPepper((0.01, 0.05), name="SaltAndPepper"),
iaa.CoarseSaltAndPepper((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarseSaltAndPepper"),
iaa.Salt((0.01, 0.05), name="Salt"),
iaa.CoarseSalt((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarseSalt"),
iaa.Pepper((0.01, 0.05), name="Pepper"),
iaa.CoarsePepper((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarsePepper"),
iaa.Invert(0.1, name="Invert"),
# ContrastNormalization
iaa.JpegCompression((50, 99), name="JpegCompression")
]
augmenters_blur = [
iaa.GaussianBlur(sigma=(1.0, 5.0), name="GaussianBlur"),
iaa.AverageBlur(k=(3, 11), name="AverageBlur"),
iaa.MedianBlur(k=(3, 11), name="MedianBlur"),
iaa.BilateralBlur(d=(3, 11), name="BilateralBlur"),
iaa.MotionBlur(k=(3, 11), name="MotionBlur")
]
augmenters_color = [
# InColorspace (deprecated)
iaa.WithColorspace(to_colorspace="HSV",
children=iaa.Noop(),
name="WithColorspace"),
iaa.AddToHueAndSaturation((-10, 10), name="AddToHueAndSaturation"),
iaa.ChangeColorspace(to_colorspace="HSV", name="ChangeColorspace"),
iaa.Grayscale((0.01, 0.99), name="Grayscale")
]
augmenters_contrast = [
iaa.GammaContrast(gamma=(0.5, 2.0), name="GammaContrast"),
iaa.SigmoidContrast(gain=(5, 20),
cutoff=(0.25, 0.75),
name="SigmoidContrast"),
iaa.LogContrast(gain=(0.7, 1.0), name="LogContrast"),
iaa.LinearContrast((0.5, 1.5), name="LinearContrast"),
iaa.AllChannelsHistogramEqualization(
name="AllChannelsHistogramEqualization"),
iaa.HistogramEqualization(to_colorspace="HSV",
name="HistogramEqualization"),
iaa.AllChannelsCLAHE(clip_limit=(2, 10),
tile_grid_size_px=(3, 11),
name="AllChannelsCLAHE"),
iaa.CLAHE(clip_limit=(2, 10),
tile_grid_size_px=(3, 11),
to_colorspace="HSV",
name="CLAHE"),
]
augmenters_convolutional = [
iaa.Convolve(np.float32([[0, 0, 0], [0, 1, 0], [0, 0, 0]]),
name="Convolve_3x3"),
iaa.Sharpen(alpha=(0.01, 0.99), lightness=(0.5, 2), name="Sharpen"),
iaa.Emboss(alpha=(0.01, 0.99), strength=(0, 2), name="Emboss"),
iaa.EdgeDetect(alpha=(0.01, 0.99), name="EdgeDetect"),
iaa.DirectedEdgeDetect(alpha=(0.01, 0.99), name="DirectedEdgeDetect")
]
augmenters_flip = [
iaa.Fliplr(1.0, name="Fliplr"),
iaa.Flipud(1.0, name="Flipud")
]
augmenters_geometric = [
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=0,
mode="constant",
cval=(0, 255),
name="Affine_order_0_constant"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=1,
mode="constant",
cval=(0, 255),
name="Affine_order_1_constant"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=3,
mode="constant",
cval=(0, 255),
name="Affine_order_3_constant"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=1,
mode="edge",
cval=(0, 255),
name="Affine_order_1_edge"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=1,
mode="constant",
cval=(0, 255),
backend="skimage",
name="Affine_order_1_constant_skimage"),
# TODO AffineCv2
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=4,
nb_cols=4,
order=1,
mode="constant",
name="PiecewiseAffine_4x4_order_1_constant"),
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=4,
nb_cols=4,
order=0,
mode="constant",
name="PiecewiseAffine_4x4_order_0_constant"),
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=4,
nb_cols=4,
order=1,
mode="edge",
name="PiecewiseAffine_4x4_order_1_edge"),
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=8,
nb_cols=8,
order=1,
mode="constant",
name="PiecewiseAffine_8x8_order_1_constant"),
iaa.PerspectiveTransform(scale=(0.01, 0.05),
keep_size=False,
name="PerspectiveTransform"),
iaa.PerspectiveTransform(scale=(0.01, 0.05),
keep_size=True,
name="PerspectiveTransform_keep_size"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=0,
mode="constant",
cval=0,
name="ElasticTransformation_order_0_constant"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=1,
mode="constant",
cval=0,
name="ElasticTransformation_order_1_constant"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=1,
mode="nearest",
cval=0,
name="ElasticTransformation_order_1_nearest"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=1,
mode="reflect",
cval=0,
name="ElasticTransformation_order_1_reflect"),
iaa.Rot90((1, 3), keep_size=False, name="Rot90"),
iaa.Rot90((1, 3), keep_size=True, name="Rot90_keep_size")
]
augmenters_segmentation = [
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=64,
interpolation="cubic",
name="Superpixels_max_size_64_cubic"),
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=64,
interpolation="linear",
name="Superpixels_max_size_64_linear"),
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=128,
interpolation="linear",
name="Superpixels_max_size_128_linear"),
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=224,
interpolation="linear",
name="Superpixels_max_size_224_linear"),
]
augmenters_size = [
iaa.Resize((0.8, 1.2), interpolation="nearest", name="Resize_nearest"),
iaa.Resize((0.8, 1.2), interpolation="linear", name="Resize_linear"),
iaa.Resize((0.8, 1.2), interpolation="cubic", name="Resize_cubic"),
iaa.CropAndPad(percent=(-0.2, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
keep_size=False,
name="CropAndPad"),
iaa.CropAndPad(percent=(-0.2, 0.2),
pad_mode="edge",
pad_cval=(0, 255),
keep_size=False,
name="CropAndPad_edge"),
iaa.CropAndPad(percent=(-0.2, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
name="CropAndPad_keep_size"),
iaa.Crop(percent=(0.05, 0.2), keep_size=False, name="Crop"),
iaa.Crop(percent=(0.05, 0.2), name="Crop_keep_size"),
iaa.Pad(percent=(0.05, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
keep_size=False,
name="Pad"),
iaa.Pad(percent=(0.05, 0.2),
pad_mode="edge",
pad_cval=(0, 255),
keep_size=False,
name="Pad_edge"),
iaa.Pad(percent=(0.05, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
name="Pad_keep_size"),
iaa.PadToFixedSize(width=width + 10,
height=height + 10,
pad_mode="constant",
pad_cval=(0, 255),
name="PadToFixedSize"),
iaa.CropToFixedSize(width=width - 10,
height=height - 10,
name="CropToFixedSize"),
iaa.KeepSizeByResize(iaa.CropToFixedSize(height=height - 10,
width=width - 10),
interpolation="nearest",
name="KeepSizeByResize_CropToFixedSize_nearest"),
iaa.KeepSizeByResize(iaa.CropToFixedSize(height=height - 10,
width=width - 10),
interpolation="linear",
name="KeepSizeByResize_CropToFixedSize_linear"),
iaa.KeepSizeByResize(iaa.CropToFixedSize(height=height - 10,
width=width - 10),
interpolation="cubic",
name="KeepSizeByResize_CropToFixedSize_cubic"),
]
augmenters_weather = [
iaa.FastSnowyLandscape(lightness_threshold=(100, 255),
lightness_multiplier=(1.0, 4.0),
name="FastSnowyLandscape"),
iaa.Clouds(name="Clouds"),
iaa.Fog(name="Fog"),
iaa.CloudLayer(intensity_mean=(196, 255),
intensity_freq_exponent=(-2.5, -2.0),
intensity_coarse_scale=10,
alpha_min=0,
alpha_multiplier=(0.25, 0.75),
alpha_size_px_max=(2, 8),
alpha_freq_exponent=(-2.5, -2.0),
sparsity=(0.8, 1.0),
density_multiplier=(0.5, 1.0),
name="CloudLayer"),
iaa.Snowflakes(name="Snowflakes"),
iaa.SnowflakesLayer(density=(0.005, 0.075),
density_uniformity=(0.3, 0.9),
flake_size=(0.2, 0.7),
flake_size_uniformity=(0.4, 0.8),
angle=(-30, 30),
speed=(0.007, 0.03),
blur_sigma_fraction=(0.0001, 0.001),
name="SnowflakesLayer")
]
augmenters = augmenters_meta + augmenters_arithmetic + augmenters_blur + augmenters_color + augmenters_contrast \
+ augmenters_convolutional + augmenters_flip + augmenters_geometric + augmenters_segmentation \
+ augmenters_size + augmenters_weather
if only_augmenters is not None:
augmenters_reduced = []
for augmenter in augmenters:
if any([
re.search(pattern, augmenter.name)
for pattern in only_augmenters
]):
augmenters_reduced.append(augmenter)
augmenters = augmenters_reduced
return augmenters
'''
WEIGHT_CLASS_START = np.array([1, 30])
WEIGHT_CLASS_END = np.array([1, 30])
BALANCE_RATIO = 0.93
CLASS_WEIGHT = {0: 1., 1: 3}
LABEL_SMOOTHING = 0.1
from imgaug import augmenters as iaa
IMGAUG_TRAIN_SEQ = iaa.Sequential([
# iaa.CropAndPad(percent=(-0.04, 0.04)),
iaa.Fliplr(0.5), # horizontally flip 50% of the images
iaa.Flipud(0.2), # horizontally flip 50% of the images
iaa.Sometimes(0.9, iaa.ContrastNormalization((0.9, 1.1))),
iaa.Sometimes(0.9, iaa.Add((-6, 6))),
iaa.Sometimes(
0.9,
iaa.Affine(
scale=(0.98, 1.02),
translate_percent={
"x": (-0.06, 0.06),
"y": (-0.06, 0.06)
},
rotate=(-15, 15), # rotate by -10 to +10 degrees
)),
])
BATCH_SIZE_TRAIN, BATCH_SIZE_VALID = 32, 64
USE_MULTIPROCESSING = True
),
Affine(
shear=(-10, 10),
order=[0, 1],
cval=125,
),
]),
],
# do all of the above augmentations in random order
random_order=True)
COLOR = iaa.Sequential(
[
iaa.OneOf( # apply one color transformation
[
iaa.Add((0, 0)), # identity transform
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.Invert(0.05, per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.contrast.LinearContrast((0.5, 2.0), per_channel=0.5),
])
def stochastic_image_augmentation(image):
sometimes = lambda aug: iaa.Sometimes(
0.4, aug) #apply the augmenter in x% of cases
more_often = lambda aug: iaa.Sometimes(0.6, aug)
seq = iaa.Sequential(
[
iaa.OneOf([
iaa.Fliplr(p=0.8), # horizontally flip 60% of all images
iaa.Flipud(p=0.8), # vertically flip 50% of all images
]),
# crop images by -5% to 10% of their height/width
more_often(iaa.CropAndPad(percent=(-0.05, 0.1), pad_cval=(0, 255))
),
more_often(
iaa.OneOf([
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 2.0)), # emboss images
])),
sometimes(
iaa.OneOf([
iaa.Add((-10, 10), per_channel=0.5),
iaa.LinearContrast(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
iaa.GammaContrast(
(0.03, 0.15),
per_channel=0.5), # improve or worsen the contrast
])),
sometimes(
iaa.OneOf([
iaa.GaussianBlur(
(0,
3.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(
k=(2, 7)
), # blur image using local means with kernel sizes between 2 and 7
iaa.Dropout((0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200)),
])),
sometimes(
iaa.Affine(
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
}, # scale images to 80-120% of their size, individually per axis
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
}, # translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[
0,
1
], # use nearest neighbour or bilinear interpolation (fast)
cval=(
0, 255
), # if mode is constant, use a cval between 0 and 255
fit_output=True,
)),
],
random_order=True)
#seq_det = seq.to_deterministic() I think this is used if we want to apply the same augmentation of each batch
return seq.augment_image(
image) # return for each image another combinaison
def grayscale_augmenter(cfg: AugmenterConfig):
return iaa.Sequential([
iaa.Resize({'shorter-side': 'keep-aspect-ratio', 'longer-side': max(cfg.width, cfg.height)}),
iaa.PadToFixedSize(width=cfg.width, height=cfg.height, position=cfg.position, pad_mode=cfg.pad_mode),
iaa.Sometimes(0.5, iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
rotate=(-45, 45),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255),
mode='edge'
)),
iaa.SomeOf((0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
#iaa.Sometimes(0.5,
# iaa.Superpixels(
# p_replace=(0, 1.0),
# n_segments=(20, 200)
# )
# ),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
iaa.Sometimes(0.5, iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(
alpha=(0, 0.7), direction=(0.0, 1.0)
),
])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5
),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout(
(0.03, 0.15), size_percent=(0.02, 0.05),
per_channel=0.2
),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
iaa.Invert(0.05, per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.LinearContrast((0.5, 2.0), per_channel=0.5),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
#iaa.Sometimes(0.5,
# iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
# ),
# In some images distort local areas with varying strength.
#iaa.Sometimes(0.5, iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True)#,
# iaa.Grayscale()
])
def customizedImgAug(input_img):
rarely = lambda aug: iaa.Sometimes(0.1, aug)
sometimes = lambda aug: iaa.Sometimes(0.25, aug)
often = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
iaa.Fliplr(0.5),
often(iaa.Affine(
scale={"x": (0.9, 1.1), "y": (0.9, 1.1)},
translate_percent={"x": (-0.1, 0.1), "y": (-0.12, 0)},
rotate=(-10, 10),
shear=(-8, 8),
order=[0, 1],
cval=(0, 255),
)),
iaa.SomeOf((0, 4), [
rarely(
iaa.Superpixels(
p_replace=(0, 0.3),
n_segments=(20, 200)
)
),
iaa.OneOf([
iaa.GaussianBlur((0, 2.0)),
iaa.AverageBlur(k=(2, 4)),
iaa.MedianBlur(k=(3, 5)),
]),
iaa.Sharpen(alpha=(0, 0.3), lightness=(0.75, 1.5)),
iaa.Emboss(alpha=(0, 1.0), strength=(0, 0.5)),
rarely(iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.3)),
iaa.DirectedEdgeDetect(
alpha=(0, 0.7), direction=(0.0, 1.0)
),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5
),
iaa.OneOf([
iaa.Dropout((0.0, 0.05), per_channel=0.5),
iaa.CoarseDropout(
(0.03, 0.05), size_percent=(0.01, 0.05),
per_channel=0.2
),
]),
rarely(iaa.Invert(0.05, per_channel=True)),
often(iaa.Add((-40, 40), per_channel=0.5)),
iaa.Multiply((0.7, 1.3), per_channel=0.5),
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.03))),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 1.5), sigma=0.25)
),
], random_order=True),
iaa.Fliplr(0.5),
iaa.AddToHueAndSaturation(value=(-10, 10), per_channel=True)
], random_order=True) # apply augmenters in random order
output_img = seq.augment_image(input_img)
return output_img
])),
iaa.AdditiveGaussianNoise(
loc=0,
scale=(0.0, 0.05 * 255),
per_channel=0.5),
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout(
(0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
iaa.Invert(
0.05,
per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5),
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.ContrastNormalization(
(0.5, 2.0), per_channel=0.5),
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5),
sigma=0.25)),
sometimes(
iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
random_order=True)),
]),
],
random_order=True)
def get_diving48(data_root=os.path.join('/media', 'user', 'disk0'),
clip_length=8,
clip_size=256,
val_clip_length=None,
val_clip_size=None,
train_interval=2,
val_interval=2,
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
seed=torch.distributed.get_rank()
if torch.distributed.is_initialized() else 0,
**kwargs):
logging.debug("VideoIter:: clip_length = {}, interval = [train: {}, val: {}], seed = {}".format( \
clip_length, train_interval, val_interval, seed))
sometimes_aug = lambda aug: iaa.Sometimes(0.4, aug)
sometimes_seq = lambda aug: iaa.Sometimes(0.8, aug)
train_sampler = sampler.RandomSampling(num=clip_length,
interval=train_interval,
speed=[1.0, 1.0],
seed=(seed + 0))
train = VideoIter(
video_prefix=os.path.join(data_root, 'data', 'Diving48_videos', 'jpg'),
csv_filepath=os.path.join(data_root, 'labels', 'diving48_train.csv'),
include_timeslices=False,
sampler=train_sampler,
video_size=(clip_length, clip_size, clip_size),
video_transform=transforms.Compose(
transforms=iaa.Sequential([
iaa.Resize({
"shorter-side": 294,
"longer-side": "keep-aspect-ratio"
}),
iaa.CropToFixedSize(width=224, height=224, position='uniform'),
sometimes_seq(
iaa.Sequential([
sometimes_aug(iaa.GaussianBlur(sigma=[0.1, 0.2, 0.3])),
sometimes_aug(iaa.Add((-5, 15), per_channel=True)),
#sometimes_aug(iaa.AdditiveGaussianNoise(scale=0.03*255, per_channel=True)),
#sometimes_aug(iaa.pillike.EnhanceColor(factor=(1.2, 1.6))),
#sometimes_aug(iaa.MotionBlur([3,5,7])),
sometimes_aug(
iaa.AddToHueAndSaturation((-16, 16),
per_channel=True)),
sometimes_aug(iaa.LinearContrast((0.85, 1.115))),
sometimes_aug(
iaa.OneOf([
iaa.PerspectiveTransform(scale=(0.02, 0.05),
keep_size=True),
iaa.Rotate(rotate=(-10, 10)),
]))
])),
iaa.Fliplr(0.5)
]),
normalise=[mean, std]),
name='train',
shuffle_list_seed=(seed + 2))
# Only return train iterator
if (val_clip_length is None and val_clip_size is None):
return train
val_sampler = sampler.SequentialSampling(num=clip_length,
interval=val_interval,
fix_cursor=True,
shuffle=True)
val = VideoIter(
dataset_location=os.path.join(data_root, 'data', 'Diving48_videos',
'jpg'),
csv_filepath=os.path.join(data_root, 'labels', 'diving48_val.csv'),
include_timeslices=False,
sampler=val_sampler,
video_size=(val_clip_length, val_clip_size, val_clip_size),
video_transform=transforms.Compose(transforms=iaa.Sequential([
iaa.Resize({
"shorter-side": 294,
"longer-side": "keep-aspect-ratio"
}),
iaa.CropToFixedSize(width=val_clip_size,
height=val_clip_size,
position='center')
]),
normalise=[mean, std]),
name='val')
return (train, val)
def test_unusual_channel_numbers():
reseed()
images = [
(0, create_random_images((4, 16, 16))),
(1, create_random_images((4, 16, 16, 1))),
(2, create_random_images((4, 16, 16, 2))),
(4, create_random_images((4, 16, 16, 4))),
(5, create_random_images((4, 16, 16, 5))),
(10, create_random_images((4, 16, 16, 10))),
(20, create_random_images((4, 16, 16, 20)))
]
augs = [
iaa.Add((-5, 5), name="Add"),
iaa.AddElementwise((-5, 5), name="AddElementwise"),
iaa.AdditiveGaussianNoise(0.01*255, name="AdditiveGaussianNoise"),
iaa.Multiply((0.95, 1.05), name="Multiply"),
iaa.Dropout(0.01, name="Dropout"),
iaa.CoarseDropout(0.01, size_px=6, name="CoarseDropout"),
iaa.Invert(0.01, per_channel=True, name="Invert"),
iaa.GaussianBlur(sigma=(0.95, 1.05), name="GaussianBlur"),
iaa.AverageBlur((3, 5), name="AverageBlur"),
iaa.MedianBlur((3, 5), name="MedianBlur"),
iaa.Sharpen((0.0, 0.1), lightness=(1.0, 1.2), name="Sharpen"),
iaa.Emboss(alpha=(0.0, 0.1), strength=(0.5, 1.5), name="Emboss"),
iaa.EdgeDetect(alpha=(0.0, 0.1), name="EdgeDetect"),
iaa.DirectedEdgeDetect(alpha=(0.0, 0.1), direction=0,
name="DirectedEdgeDetect"),
iaa.Fliplr(0.5, name="Fliplr"),
iaa.Flipud(0.5, name="Flipud"),
iaa.Affine(translate_px=(-5, 5), name="Affine-translate-px"),
iaa.Affine(translate_percent=(-0.05, 0.05),
name="Affine-translate-percent"),
iaa.Affine(rotate=(-20, 20), name="Affine-rotate"),
iaa.Affine(shear=(-20, 20), name="Affine-shear"),
iaa.Affine(scale=(0.9, 1.1), name="Affine-scale"),
iaa.PiecewiseAffine(scale=(0.001, 0.005), name="PiecewiseAffine"),
iaa.PerspectiveTransform(scale=(0.01, 0.10),
name="PerspectiveTransform"),
iaa.ElasticTransformation(alpha=(0.1, 0.2), sigma=(0.1, 0.2),
name="ElasticTransformation"),
iaa.Sequential([iaa.Add((-5, 5)), iaa.AddElementwise((-5, 5))]),
iaa.SomeOf(1, [iaa.Add((-5, 5)), iaa.AddElementwise((-5, 5))]),
iaa.OneOf([iaa.Add((-5, 5)), iaa.AddElementwise((-5, 5))]),
iaa.Sometimes(0.5, iaa.Add((-5, 5)), name="Sometimes"),
iaa.Identity(name="Noop"),
iaa.BlendAlpha((0.0, 0.1), iaa.Add(10), name="BlendAlpha"),
iaa.BlendAlphaElementwise((0.0, 0.1), iaa.Add(10),
name="BlendAlphaElementwise"),
iaa.BlendAlphaSimplexNoise(iaa.Add(10), name="BlendAlphaSimplexNoise"),
iaa.BlendAlphaFrequencyNoise(exponent=(-2, 2),
foreground=iaa.Add(10),
name="BlendAlphaSimplexNoise"),
iaa.Superpixels(p_replace=0.01, n_segments=64),
iaa.Resize({"height": 4, "width": 4}, name="Resize"),
iaa.CropAndPad(px=(-10, 10), name="CropAndPad"),
iaa.Pad(px=(0, 10), name="Pad"),
iaa.Crop(px=(0, 10), name="Crop")
]
for aug in augs:
for (nb_channels, images_c) in images:
if aug.name != "Resize":
images_aug = aug.augment_images(images_c)
assert images_aug.shape == images_c.shape
image_aug = aug.augment_image(images_c[0])
assert image_aug.shape == images_c[0].shape
else:
images_aug = aug.augment_images(images_c)
image_aug = aug.augment_image(images_c[0])
if images_c.ndim == 3:
assert images_aug.shape == (4, 4, 4)
assert image_aug.shape == (4, 4)
else:
assert images_aug.shape == (4, 4, 4, images_c.shape[3])
assert image_aug.shape == (4, 4, images_c.shape[3])
