def __init__(self, config, mode="all"):
assert mode in ["train", "validation", "all"
], f"Should be train, validation or all, got {mode}"
self.config = config
self.sc = AnimalTriplet(config)
self.train = int(config["train_size"] * len(self.sc))
self.test = 1 - self.train
self.augmentation = config["augmentation"]
self.aug_factor = 0.5
self.resize = iaa.Resize(self.config["resize_to"])
if self.augmentation:
self.seq = iaa.Sequential(
[
iaa.Sometimes(self.aug_factor,
iaa.SaltAndPepper(0.01, per_channel=False)),
iaa.Sometimes(self.aug_factor,
iaa.CoarseDropout(0.01, size_percent=0.5)),
iaa.Fliplr(self.aug_factor),
iaa.Flipud(self.aug_factor),
iaa.Sometimes(self.aug_factor,
iaa.GaussianBlur(sigma=(0, 3.0))),
iaa.LinearContrast((0.75, 1.5)),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
],
random_order=True)
self.joints = [
[2, 0], # Nose - L_Eye
[2, 1], # Nose - R_Eye
[0, 3], # L_Eye - L_EarBase
[1, 4], # R_Eye - R_EarBase
[2, 8], # Nose - Throat
[8, 9], # Throat - L_F_Elbow
[8, 5], # Throat - R_F_Elbow
[9, 16], # L_F_Elbow - L_F_Knee
[16, 6], # L_F_Knee - L_F_Paw
[5, 17], # R_F_Elbow - R_F_Knee
[17, 7], # R_F_Knee - R_F_Paw
[14, 18], # L_B_Elbow - L_B_Knee
[18, 13], # L_B_Knee - L_B_Paw
[15, 19], # R_B_Elbow - R_B_Knee
[19, 13], # R_B_Knee - R_B_Paw
[10, 11], # Withers - TailBase
]
if mode != "all":
# split_indices = np.arange(self.train) if mode == "train" else np.arange(self.train + 1, len(self.sc))
dset_indices = np.arange(len(self.sc))
train_indices, test_indices = sklearn.model_selection.train_test_split(
dset_indices,
train_size=float(config["train_size"]),
random_state=int(config["random_state"]))
if mode == "train":
self.data = SubDataset(self.sc, train_indices)
else:
self.data = SubDataset(self.sc, test_indices)
else:
self.data = self.sc
def _build_augmenter(self, **kwargs):
aug = iaa.Sometimes(0.5,
iaa.CoarseDropout((0.01, 0.05), size_percent=(0.05, 0.25)))
return aug
def load_aug():
import imgaug as ia
from imgaug import augmenters as iaa
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq[0] = iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop images by -5% to 10% of their height/width
sometimes(iaa.CropAndPad(
percent=(-0.05, 0.1),
pad_mode=ia.ALL,
pad_cval=(0, 255)
)),
sometimes(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
# scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))),
# convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)),
# blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)),
# blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5),
# change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0))
)
]),
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)),
# move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
# sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True
)
],
random_order=True
)
def build_augmentation_pipeline(self,
height=None,
width=None,
apply_prob=0.5):
sometimes = lambda aug: iaa.Sometimes(apply_prob, aug)
pipeline = iaa.Sequential(random_order=False)
cfg = self.cfg
if cfg["mirror"]:
opt = cfg["mirror"] # fliplr
if type(opt) == int:
pipeline.add(sometimes(iaa.Fliplr(opt)))
else:
pipeline.add(sometimes(iaa.Fliplr(0.5)))
if cfg["rotation"] > 0:
pipeline.add(
iaa.Sometimes(
cfg["rotratio"],
iaa.Affine(rotate=(-cfg["rotation"], cfg["rotation"])),
))
if cfg["motion_blur"]:
opts = cfg["motion_blur_params"]
pipeline.add(sometimes(iaa.MotionBlur(**opts)))
if cfg["covering"]:
pipeline.add(
sometimes(
iaa.CoarseDropout(0.02, size_percent=0.3,
per_channel=0.5)))
if cfg["elastic_transform"]:
pipeline.add(sometimes(iaa.ElasticTransformation(sigma=5)))
if cfg.get("gaussian_noise", False):
opt = cfg.get("gaussian_noise", False)
if type(opt) == int or type(opt) == float:
pipeline.add(
sometimes(
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, opt),
per_channel=0.5)))
else:
pipeline.add(
sometimes(
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, 0.05 * 255),
per_channel=0.5)))
if cfg.get("grayscale", False):
pipeline.add(sometimes(iaa.Grayscale(alpha=(0.5, 1.0))))
def get_aug_param(cfg_value):
if isinstance(cfg_value, dict):
opt = cfg_value
else:
opt = {}
return opt
cfg_cnt = cfg.get("contrast", {})
cfg_cnv = cfg.get("convolution", {})
contrast_aug = ["histeq", "clahe", "gamma", "sigmoid", "log", "linear"]
for aug in contrast_aug:
aug_val = cfg_cnt.get(aug, False)
cfg_cnt[aug] = aug_val
if aug_val:
cfg_cnt[aug + "ratio"] = cfg_cnt.get(aug + "ratio", 0.1)
convolution_aug = ["sharpen", "emboss", "edge"]
for aug in convolution_aug:
aug_val = cfg_cnv.get(aug, False)
cfg_cnv[aug] = aug_val
if aug_val:
cfg_cnv[aug + "ratio"] = cfg_cnv.get(aug + "ratio", 0.1)
if cfg_cnt["histeq"]:
opt = get_aug_param(cfg_cnt["histeq"])
pipeline.add(
iaa.Sometimes(cfg_cnt["histeqratio"],
iaa.AllChannelsHistogramEqualization(**opt)))
if cfg_cnt["clahe"]:
opt = get_aug_param(cfg_cnt["clahe"])
pipeline.add(
iaa.Sometimes(cfg_cnt["claheratio"],
iaa.AllChannelsCLAHE(**opt)))
if cfg_cnt["log"]:
opt = get_aug_param(cfg_cnt["log"])
pipeline.add(
iaa.Sometimes(cfg_cnt["logratio"], iaa.LogContrast(**opt)))
if cfg_cnt["linear"]:
opt = get_aug_param(cfg_cnt["linear"])
pipeline.add(
iaa.Sometimes(cfg_cnt["linearratio"],
iaa.LinearContrast(**opt)))
if cfg_cnt["sigmoid"]:
opt = get_aug_param(cfg_cnt["sigmoid"])
pipeline.add(
iaa.Sometimes(cfg_cnt["sigmoidratio"],
iaa.SigmoidContrast(**opt)))
if cfg_cnt["gamma"]:
opt = get_aug_param(cfg_cnt["gamma"])
pipeline.add(
iaa.Sometimes(cfg_cnt["gammaratio"], iaa.GammaContrast(**opt)))
if cfg_cnv["sharpen"]:
opt = get_aug_param(cfg_cnv["sharpen"])
pipeline.add(
iaa.Sometimes(cfg_cnv["sharpenratio"], iaa.Sharpen(**opt)))
if cfg_cnv["emboss"]:
opt = get_aug_param(cfg_cnv["emboss"])
pipeline.add(
iaa.Sometimes(cfg_cnv["embossratio"], iaa.Emboss(**opt)))
if cfg_cnv["edge"]:
opt = get_aug_param(cfg_cnv["edge"])
pipeline.add(
iaa.Sometimes(cfg_cnv["edgeratio"], iaa.EdgeDetect(**opt)))
if height is not None and width is not None:
if not cfg.get("crop_by", False):
crop_by = 0.15
else:
crop_by = cfg.get("crop_by", False)
pipeline.add(
iaa.Sometimes(
cfg.get("cropratio", 0.4),
iaa.CropAndPad(percent=(-crop_by, crop_by),
keep_size=False),
))
pipeline.add(iaa.Resize({"height": height, "width": width}))
return pipeline
iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0))
)
]),
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
def test_unusual_channel_numbers():
reseed()
images = [(0, create_random_images((4, 16, 16))),
(1, create_random_images((4, 16, 16, 1))),
(2, create_random_images((4, 16, 16, 2))),
(4, create_random_images((4, 16, 16, 4))),
(5, create_random_images((4, 16, 16, 5))),
(10, create_random_images((4, 16, 16, 10))),
(20, create_random_images((4, 16, 16, 20)))]
augs = [
iaa.Add((-5, 5), name="Add"),
iaa.AddElementwise((-5, 5), name="AddElementwise"),
iaa.AdditiveGaussianNoise(0.01 * 255, name="AdditiveGaussianNoise"),
iaa.Multiply((0.95, 1.05), name="Multiply"),
iaa.Dropout(0.01, name="Dropout"),
iaa.CoarseDropout(0.01, size_px=6, name="CoarseDropout"),
iaa.Invert(0.01, per_channel=True, name="Invert"),
iaa.GaussianBlur(sigma=(0.95, 1.05), name="GaussianBlur"),
iaa.AverageBlur((3, 5), name="AverageBlur"),
iaa.MedianBlur((3, 5), name="MedianBlur"),
iaa.Sharpen((0.0, 0.1), lightness=(1.0, 1.2), name="Sharpen"),
iaa.Emboss(alpha=(0.0, 0.1), strength=(0.5, 1.5), name="Emboss"),
iaa.EdgeDetect(alpha=(0.0, 0.1), name="EdgeDetect"),
iaa.DirectedEdgeDetect(alpha=(0.0, 0.1),
direction=0,
name="DirectedEdgeDetect"),
iaa.Fliplr(0.5, name="Fliplr"),
iaa.Flipud(0.5, name="Flipud"),
iaa.Affine(translate_px=(-5, 5), name="Affine-translate-px"),
iaa.Affine(translate_percent=(-0.05, 0.05),
name="Affine-translate-percent"),
iaa.Affine(rotate=(-20, 20), name="Affine-rotate"),
iaa.Affine(shear=(-20, 20), name="Affine-shear"),
iaa.Affine(scale=(0.9, 1.1), name="Affine-scale"),
iaa.PiecewiseAffine(scale=(0.001, 0.005), name="PiecewiseAffine"),
iaa.PerspectiveTransform(scale=(0.01, 0.10),
name="PerspectiveTransform"),
iaa.ElasticTransformation(alpha=(0.1, 0.2),
sigma=(0.1, 0.2),
name="ElasticTransformation"),
iaa.Sequential([iaa.Add((-5, 5)),
iaa.AddElementwise((-5, 5))]),
iaa.SomeOf(1, [iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))]),
iaa.OneOf([iaa.Add((-5, 5)),
iaa.AddElementwise((-5, 5))]),
iaa.Sometimes(0.5, iaa.Add((-5, 5)), name="Sometimes"),
iaa.Noop(name="Noop"),
iaa.Alpha((0.0, 0.1), iaa.Add(10), name="Alpha"),
iaa.AlphaElementwise((0.0, 0.1), iaa.Add(10), name="AlphaElementwise"),
iaa.SimplexNoiseAlpha(iaa.Add(10), name="SimplexNoiseAlpha"),
iaa.FrequencyNoiseAlpha(exponent=(-2, 2),
first=iaa.Add(10),
name="SimplexNoiseAlpha"),
iaa.Superpixels(p_replace=0.01, n_segments=64),
iaa.Resize({
"height": 4,
"width": 4
}, name="Resize"),
iaa.CropAndPad(px=(-10, 10), name="CropAndPad"),
iaa.Pad(px=(0, 10), name="Pad"),
iaa.Crop(px=(0, 10), name="Crop")
]
for aug in augs:
for (nb_channels, images_c) in images:
if aug.name != "Resize":
images_aug = aug.augment_images(images_c)
assert images_aug.shape == images_c.shape
image_aug = aug.augment_image(images_c[0])
assert image_aug.shape == images_c[0].shape
else:
images_aug = aug.augment_images(images_c)
image_aug = aug.augment_image(images_c[0])
if images_c.ndim == 3:
assert images_aug.shape == (4, 4, 4)
assert image_aug.shape == (4, 4)
else:
assert images_aug.shape == (4, 4, 4, images_c.shape[3])
assert image_aug.shape == (4, 4, images_c.shape[3])
def get_augmenter(imgs, probs=0.5):
# Sometimes(0.5, ...) applies the given augmenter in 50% of all cases,
sometimes = lambda aug: iaa.Sometimes(probs, aug)
# Define our sequence of augmentation steps that will be applied to every image
augment_sequences = iaa.Sequential(
[
# # apply the following augmenters to most images
sometimes(iaa.Affine(
scale={"x": (0.9, 1.1), "y": (0.9, 1.1)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.12, 0.12), "y": (-0.12, 0.12)}, # translate by -20 to +20 percent (per axis)
rotate=(-5, 5), # rotate by -45 to +45 degrees
shear=(-13, 13), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode="constant" # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute one of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
# sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 2.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 4)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.05), per_channel=0.5), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.06), size_percent=(0.02, 0.05), per_channel=0.2),
]),
# iaa.Invert(0.05, per_channel=True), # invert color channels
# iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
# iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.LinearContrast((0.5, 2.0))
)
]),
# iaa.LinearContrast((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
# iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True
)
],
random_order=True
)
images_aug = augment_sequences(images=imgs)
return images_aug
def _load_augmentation_aug_all():
""" Load image augmentation model """
def sometimes(aug):
return iaa.Sometimes(0.5, aug) # 0.5 chance to perform the augment
return iaa.Sequential(
[
# apply the following augmenters to most images
# 1. horizontally flip 50% of all images
iaa.Fliplr(0.5),
# 2. vertically flip 20% of all images
iaa.Flipud(0.2),
# 3. crop images by -5% to 10% of their height/width
sometimes(
iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode='constant',
pad_cval=(0, 255))),
# 4. affine the image by:
sometimes(
iaa.Affine(
# 4.1. scale images to 80-120% of their size, individually per axis
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
# 4.2. translate by -20 to +20 percent (per axis)
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
# 4.3. rotate by -45 to +45 degrees
rotate=(-45, 45),
# 4.4. shear by -16 to +16 degrees
shear=(-16, 16),
# 4.5. use nearest neighbour or bilinear interpolation (fast)
order=[0, 1],
# 4.6. if mode is constant, use a cval between 0 and 255
cval=(0, 255),
# use any of scikit-image's warping modes
# (see 2nd image from the top for examples)
mode='constant')),
# execute 0 to 5 of the following (less important) augmenters per
# image don't execute all of them, as that would often be way too
# strong
iaa.SomeOf(
(0, 5),
[
# S1. convert images into their superpixel representation
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
# S2. image blur (randomly choose one of the three blurs)
iaa.OneOf([
# S2.1. Gaussian blur with a sigma between 0 and 3.0
iaa.GaussianBlur((0, 3.0)),
# S2.2. average blur (local means) with kernel sizes between 2 and 7
iaa.AverageBlur(k=(2, 7)),
# S2.3. median blur (local medians) with kernel sizes between 3 and 11
iaa.MedianBlur(k=(3, 11)),
]),
# S3. sharpen images
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# S4. emboss images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# S5. using a blobby mask to blend the edges (either for all edges or for directed edges)
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
# S6. add gaussian noise to images
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# S7. dropout pixels
iaa.OneOf([
# randomly remove up to 10% of the pixels
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
# S8. invert color channels
iaa.Invert(0.05, per_channel=True),
# S9. change brightness of images (by -10 to 10 of original value)
iaa.Add((-10, 10), per_channel=0.5),
# S10. change hue and saturation
iaa.AddToHueAndSaturation((-20, 20)),
# S11. change the brightness of
# the whole image (sometimes per channel)
# or of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0)))
]),
# S12. improve or worsen the contrast
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),
# S13. grayscale
iaa.Grayscale(alpha=(0.0, 1.0)),
# S14. move pixels locally around (with random strengths)
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# S15. move parts of the image around
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
# S16. four point perspective transformations
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
],
random_order=True)
def coarse_dropout(image):
size_percent = np.random.rand() * 0.2
image_aug = iaa.CoarseDropout(0.02,
size_percent=size_percent,
per_channel=0.5)(image=image)
return image_aug
def __init__(self, **config):
from numpy.random import uniform
from numpy.random import randint
## old photometric
self.aug = iaa.Sequential([
iaa.Sometimes(0.25, iaa.GaussianBlur(sigma=(0, 3.0))),
iaa.Sometimes(
0.25,
iaa.OneOf([
iaa.Dropout(p=(0, 0.1)),
iaa.CoarseDropout(0.1, size_percent=0.5)
])),
iaa.Sometimes(
0.25,
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, 0.05),
per_channel=0.5),
)
])
if config['photometric']['enable']:
params = config['photometric']['params']
aug_all = []
if params.get('random_brightness', False):
change = params['random_brightness']['max_abs_change']
aug = iaa.Add((-change, change))
# aug_all.append(aug)
aug_all.append(aug)
# if params['random_contrast']:
if params.get('random_contrast', False):
change = params['random_contrast']['strength_range']
aug = iaa.LinearContrast((change[0], change[1]))
aug_all.append(aug)
# if params['additive_gaussian_noise']:
if params.get('additive_gaussian_noise', False):
change = params['additive_gaussian_noise']['stddev_range']
aug = iaa.AdditiveGaussianNoise(scale=(change[0], change[1]))
aug_all.append(aug)
# if params['additive_speckle_noise']:
if params.get('additive_speckle_noise', False):
change = params['additive_speckle_noise']['prob_range']
# aug = iaa.Dropout(p=(change[0], change[1]))
aug = iaa.ImpulseNoise(p=(change[0], change[1]))
aug_all.append(aug)
# if params['motion_blur']:
if params.get('motion_blur', False):
change = params['motion_blur']['max_kernel_size']
if change > 3:
change = randint(3, change)
elif change == 3:
aug = iaa.Sometimes(0.5, iaa.MotionBlur(change))
aug_all.append(aug)
if params.get('GaussianBlur', False):
change = params['GaussianBlur']['sigma']
aug = iaa.GaussianBlur(sigma=(change))
aug_all.append(aug)
self.aug = iaa.Sequential(aug_all)
else:
self.aug = iaa.Sequential([
iaa.Noop(),
])
def get():
def sometimes(aug): return iaa.Sometimes(0.5, aug)
return iaa.Sequential(
[
# apply the following augmenters to most images
# iaa.Fliplr(0.5), # horizontally flip 50% of all images
# iaa.Flipud(0.2), # vertically flip 20% of all images
# crop images by -5% to 10% of their height/width
sometimes(iaa.CropAndPad(
percent=(-0.05, 0.1),
pad_mode=ia.ALL,
pad_cval=(0, 255)
)),
sometimes(iaa.Affine(
# scale images to 90-110% of their size, individually per axis
scale={'x': (0.9, 1.1), 'y': (0.9, 1.1)},
# translate by -10 to +10 percent (per axis)
translate_percent={'x': (-0.1, 0.1), 'y': (-0.1, 0.1)},
rotate=(-20, 20), # rotate by -20 to +20 degrees
# shear=(-16, 16), # shear by -16 to +16 degrees
# use nearest neighbour or bilinear interpolation (fast)
order=[0, 1],
# if mode is constant, use a cval between 0 and 255
cval=(0, 255),
# use any of scikit-image's warping modes (see 2nd image from the top for examples)
mode=ia.ALL
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
# convert images into their superpixel representation
sometimes(iaa.Superpixels(
p_replace=(0, 1.0), n_segments=(20, 200))),
iaa.OneOf([
# blur images with a sigma between 0 and 3.0
# 使用0和1.5之間的sigma模糊圖像
iaa.GaussianBlur((0, 1.5)),
# blur image using local means with kernel sizes between 2 and 7
# 使用局部大小在2到5之間的內核來模糊圖像
iaa.AverageBlur(k=(2, 5)),
# blur image using local medians with kernel sizes between 2 and 7
# 使用內核大小在2到7之間的局部中值模糊圖像
# iaa.MedianBlur(k=(2, 7)),
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(
0.75, 1.5)), # sharpen images 銳化圖像
# iaa.Emboss(alpha=(0, 1.0), strength=(
# 0, 2.0)), # emboss images 浮雕圖像
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
# 搜索所有邊緣或有向邊緣，使用過大的蒙版將結果與原始圖像融合
# add gaussian noise to images
# 給圖像增加高斯噪聲
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05 * 255), per_channel=0.3),
iaa.OneOf([
# randomly remove up to 10% of the pixels
# 隨機刪除多達10％的像素
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15), size_percent=(
0.02, 0.05), per_channel=0.2),
]),
# invert color channels
# 反轉顏色通道
# iaa.Invert(0.05, per_channel=True),
# change brightness of images (by -10 to 10 of original value)
# 更改圖像的亮度（原始值的-10到10）
iaa.Add((-5, 5), per_channel=0.5),
# change hue and saturation
# 改變色調和飽和度
iaa.AddToHueAndSaturation((-10, 10)),
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
# 更改整個圖像的亮度（有時每通道）或更改子區域的亮度
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0))
)
]),
# improve or worsen the contrast
# 提升或降低對比度
iaa.ContrastNormalization((0.5, 1.0), per_channel=0.5),
iaa.Grayscale(alpha=(0.0, 0.5)),
# move pixels locally around (with random strengths)
# 局部移動像素（具有隨機強度）
sometimes(iaa.ElasticTransformation(
alpha=(0.5, 3.5), sigma=0.25)),
# sometimes move parts of the image around
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True
)
],
random_order=True
)
#branchConfig = [["Steer", "Gas", "Brake"], ["Steer", "Gas", "Brake"], \
# ["Steer", "Gas", "Brake"], ["Steer", "Gas", "Brake"],
# ["Speed"] # turn of speed branch for the moment
# ]
## GPU configuration
st = lambda aug: iaa.Sometimes(0.4, aug)
oc = lambda aug: iaa.Sometimes(0.3, aug)
rl = lambda aug: iaa.Sometimes(0.09, aug)
seq = iaa.Sequential([
rl(iaa.GaussianBlur((0, 1.5))), # blur images with a sigma between 0 and 1.5
rl(iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05), per_channel=0.5)), # add gaussian noise to images
oc(iaa.Dropout((0.0, 0.10), per_channel=0.5)), # randomly remove up to X% of the pixels
oc(iaa.CoarseDropout((0.0, 0.10), size_percent=(0.08, 0.2),per_channel=0.5)), # randomly remove up to X% of the pixels
oc(iaa.Add((-40, 40), per_channel=0.5)), # change brightness of images (by -X to Y of original value)
st(iaa.Multiply((0.10, 2.5), per_channel=0.2)), # change brightness of images (X-Y% of original value)
rl(iaa.ContrastNormalization((0.5, 1.5), per_channel=0.5)), # improve or worsen the contrast
#rl(iaa.Grayscale((0.0, 1))), # put grayscale
], random_order=True)
# branch_indices = [x for x in range(len(branchConfig))]
# Prepare data generators
batchListGenTrain = []
batchListGenVal = []
batchListName = []
# for i in range(1):
# with tf.name_scope("Branch_" + str(i)):
# if branchConfig[i][0] == "Speed":
def get_processing_sequence(image_size):
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
return iaa.Sequential([
# iaa.Resize({"longer-side": image_size, "shorter-side": "keep-aspect-ratio"}),
# iaa.PadToFixedSize(width=image_size, height=image_size),
iaa.Resize(image_size),
iaa.Fliplr(0.3),
iaa.Flipud(0.3),
iaa.Affine(
scale={"x": (0.8, 1.0), "y": (0.8, 1.0)},
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
rotate=(-45, 45),
shear=(-20, 20),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL
),
# iaa.ChangeColorTemperature(kelvin=(1000, 11000)),
iaa.SomeOf((0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
sometimes(
iaa.Superpixels(
p_replace=(0, 1.0),
n_segments=(20, 200)
)
),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
sometimes(iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(
alpha=(0, 0.7), direction=(0.0, 1.0)
),
])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5
),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout(
(0.03, 0.15), size_percent=(0.02, 0.05),
per_channel=0.2
),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
iaa.Invert(0.05, per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.2), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.LinearContrast((0.5, 1.5), per_channel=0.5),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True)
])
def __getitem__(self, idx):
item_path = self.items[self.indices[idx]]
relative_path_split = item_path[len(self.root_dir):].split("/")
action = relative_path_split[0]
puppet_mask_file = self.root_dir + "puppet_mask/" + relative_path_split[0] + "/" + relative_path_split[1] + "/puppet_tensor.pt"
puppet_corners = torch.load(puppet_mask_file)
label = sio.loadmat(item_path + "/joint_positions")
poses = torch.from_numpy(label["pos_img"].T).float()
all_frames = sorted(glob.glob(item_path + "/*.png"))
images = []
for image_path in all_frames:
images.append(torch.from_numpy(io.imread(image_path)).int())
image_height = len(images[0])
image_width = len(images[0][0])
self.set_augmentation_parameters()
#self.calc_bbox_and_center(image_width, image_height)
processed_frames = []
processed_poses = []
trans_matrices = []
bounding_boxes = []
original_window_sizes = []
current_frame = 0
for frame, pose in zip(images, poses):
bbox = torch.IntTensor(4)
bbox[0] = puppet_corners[current_frame, 0]
bbox[1] = puppet_corners[current_frame, 2]
bbox[2] = puppet_corners[current_frame, 1]
bbox[3] = puppet_corners[current_frame, 3]
bbox_width = torch.abs(bbox[0] - bbox[2]).item()
bbox_height = torch.abs(bbox[1] - bbox[3]).item()
self.original_window_size = torch.IntTensor([max(bbox_height, bbox_width), max(bbox_height, bbox_width)])
if self.use_random_parameters:
offset = 40
else:
offset = 30
if self.use_gt_bb:
self.calc_bbox_and_center(image_width, image_height, pre_bb=bbox, offset=offset)
else:
self.calc_bbox_and_center(image_width, image_height)
current_frame = current_frame + 1
trans_matrix, norm_frame, norm_pose = self.preprocess(frame, pose)
norm_pose = self.map_to_mpii(norm_pose)
norm_pose = self.set_visibility(norm_pose)
if self.aug_conf["flip"]:
norm_pose = flip_lr_pose(norm_pose)
bounding_boxes.append(self.bbox.clone().unsqueeze(0))
processed_poses.append(norm_pose.unsqueeze(0))
if self.use_random_parameters:
processed_frames.append(norm_frame.numpy())
else:
processed_frames.append(norm_frame.unsqueeze(0))
trans_matrices.append(trans_matrix.clone().unsqueeze(0))
original_window_sizes.append(self.original_window_size.clone().unsqueeze(0))
number_of_frames = len(processed_frames)
# further augmentation, beyond Luvizon et al.
if self.use_random_parameters:
processed_frames = list(map(lambda x: (x + 1) / 2.0, processed_frames))
seq = iaa.Sequential(
[
#iaa.Sometimes(1.0, iaa.MotionBlur(k=3, angle=self.motion_blur_angle)),
iaa.Sequential([
iaa.Sometimes(self.further_augmentation_prob, iaa.CoarseDropout((0.05 * self.aug_conf["scale"]).item(), size_percent=(self.dropout_size_percent * self.aug_conf["scale"]).item())),
iaa.Sometimes(self.further_augmentation_prob, iaa.SaltAndPepper((self.salt_and_pepper * self.aug_conf["scale"]).item())),
], random_order=True)
]
, random_order=False)
np_processed_frames = seq(images=processed_frames)
processed_frames = list(map(lambda x: 2.0 * torch.FloatTensor(np.expand_dims(x, axis=0)) - 1, np_processed_frames))
frames = torch.cat(processed_frames)
poses = torch.cat(processed_poses)
trans_matrices = torch.cat(trans_matrices)
t_bounding_boxes = torch.cat(bounding_boxes)
original_window_sizes = torch.cat(original_window_sizes)
trans_matrices, frames, poses, t_bounding_boxes, original_window_sizes = self.apply_padding(trans_matrices, frames, poses, t_bounding_boxes, original_window_sizes)
frames = frames.permute(0, 3, 1, 2)
action_1h = torch.zeros(21).float()
action_1h[self.action_mapping[action]] = 1
t_sequence_length = torch.ByteTensor([number_of_frames])
t_index = torch.zeros(1).float()
t_index[0] = idx
t_parameters = torch.zeros(5).float()
t_parameters[0] = self.aug_conf["scale"]
t_parameters[1] = float(self.aug_conf["angle"])
t_parameters[2] = float(self.aug_conf["flip"])
t_parameters[3] = float(self.aug_conf["trans_x"])
t_parameters[4] = float(self.aug_conf["trans_y"])
return {
"action_1h": action_1h,
"action_label": action,
"normalized_frames": frames,
"normalized_poses": poses,
"sequence_length": t_sequence_length,
"trans_matrices": trans_matrices,
"parameters": t_parameters,
"original_window_size": original_window_sizes,
"index": t_index,
"bbox": t_bounding_boxes,
"all_frames": all_frames
}
def test_keypoint_augmentation():
reseed()
keypoints = []
for y in sm.xrange(40 // 5):
for x in sm.xrange(60 // 5):
keypoints.append(ia.Keypoint(y=y * 5, x=x * 5))
keypoints_oi = ia.KeypointsOnImage(keypoints, shape=(40, 60, 3))
keypoints_oi_empty = ia.KeypointsOnImage([], shape=(40, 60, 3))
augs = [
iaa.Add((-5, 5), name="Add"),
iaa.AddElementwise((-5, 5), name="AddElementwise"),
iaa.AdditiveGaussianNoise(0.01 * 255, name="AdditiveGaussianNoise"),
iaa.Multiply((0.95, 1.05), name="Multiply"),
iaa.Dropout(0.01, name="Dropout"),
iaa.CoarseDropout(0.01, size_px=6, name="CoarseDropout"),
iaa.Invert(0.01, per_channel=True, name="Invert"),
iaa.GaussianBlur(sigma=(0.95, 1.05), name="GaussianBlur"),
iaa.AverageBlur((3, 5), name="AverageBlur"),
iaa.MedianBlur((3, 5), name="MedianBlur"),
iaa.Sharpen((0.0, 0.1), lightness=(1.0, 1.2), name="Sharpen"),
iaa.Emboss(alpha=(0.0, 0.1), strength=(0.5, 1.5), name="Emboss"),
iaa.EdgeDetect(alpha=(0.0, 0.1), name="EdgeDetect"),
iaa.DirectedEdgeDetect(alpha=(0.0, 0.1),
direction=0,
name="DirectedEdgeDetect"),
iaa.Fliplr(0.5, name="Fliplr"),
iaa.Flipud(0.5, name="Flipud"),
iaa.Affine(translate_px=(-5, 5), name="Affine-translate-px"),
iaa.Affine(translate_percent=(-0.05, 0.05),
name="Affine-translate-percent"),
iaa.Affine(rotate=(-20, 20), name="Affine-rotate"),
iaa.Affine(shear=(-20, 20), name="Affine-shear"),
iaa.Affine(scale=(0.9, 1.1), name="Affine-scale"),
iaa.PiecewiseAffine(scale=(0.001, 0.005), name="PiecewiseAffine"),
iaa.ElasticTransformation(alpha=(0.1, 0.2),
sigma=(0.1, 0.2),
name="ElasticTransformation"),
iaa.Alpha((0.0, 0.1), iaa.Add(10), name="Alpha"),
iaa.AlphaElementwise((0.0, 0.1), iaa.Add(10), name="AlphaElementwise"),
iaa.SimplexNoiseAlpha(iaa.Add(10), name="SimplexNoiseAlpha"),
iaa.FrequencyNoiseAlpha(exponent=(-2, 2),
first=iaa.Add(10),
name="SimplexNoiseAlpha"),
iaa.Superpixels(p_replace=0.01, n_segments=64),
iaa.Resize(0.5, name="Resize"),
iaa.CropAndPad(px=(-10, 10), name="CropAndPad"),
iaa.Pad(px=(0, 10), name="Pad"),
iaa.Crop(px=(0, 10), name="Crop")
]
for aug in augs:
dss = []
for i in sm.xrange(10):
aug_det = aug.to_deterministic()
kp_fully_empty_aug = aug_det.augment_keypoints([])
assert kp_fully_empty_aug == []
kp_first_empty_aug = aug_det.augment_keypoints(keypoints_oi_empty)
assert len(kp_first_empty_aug.keypoints) == 0
kp_image = keypoints_oi.to_keypoint_image(size=5)
kp_image_aug = aug_det.augment_image(kp_image)
kp_image_aug_rev = ia.KeypointsOnImage.from_keypoint_image(
kp_image_aug,
if_not_found_coords={
"x": -9999,
"y": -9999
},
nb_channels=1)
kp_aug = aug_det.augment_keypoints([keypoints_oi])[0]
ds = []
assert len(kp_image_aug_rev.keypoints) == len(kp_aug.keypoints), (
"Lost keypoints for '%s' (%d vs expected %d)" %
(aug.name, len(
kp_aug.keypoints), len(kp_image_aug_rev.keypoints)))
gen = zip(kp_aug.keypoints, kp_image_aug_rev.keypoints)
for kp_pred, kp_pred_img in gen:
kp_pred_lost = (kp_pred.x == -9999 and kp_pred.y == -9999)
kp_pred_img_lost = (kp_pred_img.x == -9999
and kp_pred_img.y == -9999)
if not kp_pred_lost and not kp_pred_img_lost:
d = np.sqrt((kp_pred.x - kp_pred_img.x)**2 +
(kp_pred.y - kp_pred_img.y)**2)
ds.append(d)
dss.extend(ds)
if len(ds) == 0:
print("[INFO] No valid keypoints found for '%s' "
"in test_keypoint_augmentation()" % (str(aug), ))
assert np.average(dss) < 5.0, \
"Average distance too high (%.2f, with ds: %s)" \
% (np.average(dss), str(dss))
def create_augmenters(height, width, height_augmentable, width_augmentable,
only_augmenters):
def lambda_func_images(images, random_state, parents, hooks):
return images
def lambda_func_heatmaps(heatmaps, random_state, parents, hooks):
return heatmaps
def lambda_func_keypoints(keypoints, random_state, parents, hooks):
return keypoints
def assertlambda_func_images(images, random_state, parents, hooks):
return True
def assertlambda_func_heatmaps(heatmaps, random_state, parents, hooks):
return True
def assertlambda_func_keypoints(keypoints, random_state, parents, hooks):
return True
augmenters_meta = [
iaa.Sequential([iaa.Noop(), iaa.Noop()],
random_order=False,
name="Sequential_2xNoop"),
iaa.Sequential([iaa.Noop(), iaa.Noop()],
random_order=True,
name="Sequential_2xNoop_random_order"),
iaa.SomeOf((1, 3),
[iaa.Noop(), iaa.Noop(), iaa.Noop()],
random_order=False,
name="SomeOf_3xNoop"),
iaa.SomeOf((1, 3),
[iaa.Noop(), iaa.Noop(), iaa.Noop()],
random_order=True,
name="SomeOf_3xNoop_random_order"),
iaa.OneOf([iaa.Noop(), iaa.Noop(), iaa.Noop()], name="OneOf_3xNoop"),
iaa.Sometimes(0.5, iaa.Noop(), name="Sometimes_Noop"),
iaa.WithChannels([1, 2], iaa.Noop(), name="WithChannels_1_and_2_Noop"),
iaa.Noop(name="Noop"),
iaa.Lambda(func_images=lambda_func_images,
func_heatmaps=lambda_func_heatmaps,
func_keypoints=lambda_func_keypoints,
name="Lambda"),
iaa.AssertLambda(func_images=assertlambda_func_images,
func_heatmaps=assertlambda_func_heatmaps,
func_keypoints=assertlambda_func_keypoints,
name="AssertLambda"),
iaa.AssertShape((None, height_augmentable, width_augmentable, None),
name="AssertShape"),
iaa.ChannelShuffle(0.5, name="ChannelShuffle")
]
augmenters_arithmetic = [
iaa.Add((-10, 10), name="Add"),
iaa.AddElementwise((-10, 10), name="AddElementwise"),
#iaa.AddElementwise((-500, 500), name="AddElementwise"),
iaa.AdditiveGaussianNoise(scale=(5, 10), name="AdditiveGaussianNoise"),
iaa.AdditiveLaplaceNoise(scale=(5, 10), name="AdditiveLaplaceNoise"),
iaa.AdditivePoissonNoise(lam=(1, 5), name="AdditivePoissonNoise"),
iaa.Multiply((0.5, 1.5), name="Multiply"),
iaa.MultiplyElementwise((0.5, 1.5), name="MultiplyElementwise"),
iaa.Dropout((0.01, 0.05), name="Dropout"),
iaa.CoarseDropout((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarseDropout"),
iaa.ReplaceElementwise((0.01, 0.05), (0, 255),
name="ReplaceElementwise"),
#iaa.ReplaceElementwise((0.95, 0.99), (0, 255), name="ReplaceElementwise"),
iaa.SaltAndPepper((0.01, 0.05), name="SaltAndPepper"),
iaa.ImpulseNoise((0.01, 0.05), name="ImpulseNoise"),
iaa.CoarseSaltAndPepper((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarseSaltAndPepper"),
iaa.Salt((0.01, 0.05), name="Salt"),
iaa.CoarseSalt((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarseSalt"),
iaa.Pepper((0.01, 0.05), name="Pepper"),
iaa.CoarsePepper((0.01, 0.05),
size_percent=(0.01, 0.1),
name="CoarsePepper"),
iaa.Invert(0.1, name="Invert"),
# ContrastNormalization
iaa.JpegCompression((50, 99), name="JpegCompression")
]
augmenters_blend = [
iaa.Alpha((0.01, 0.99), iaa.Noop(), name="Alpha"),
iaa.AlphaElementwise((0.01, 0.99), iaa.Noop(),
name="AlphaElementwise"),
iaa.SimplexNoiseAlpha(iaa.Noop(), name="SimplexNoiseAlpha"),
iaa.FrequencyNoiseAlpha((-2.0, 2.0),
iaa.Noop(),
name="FrequencyNoiseAlpha")
]
augmenters_blur = [
iaa.GaussianBlur(sigma=(1.0, 5.0), name="GaussianBlur"),
iaa.AverageBlur(k=(3, 11), name="AverageBlur"),
iaa.MedianBlur(k=(3, 11), name="MedianBlur"),
iaa.BilateralBlur(d=(3, 11), name="BilateralBlur"),
iaa.MotionBlur(k=(3, 11), name="MotionBlur")
]
augmenters_color = [
# InColorspace (deprecated)
iaa.WithColorspace(to_colorspace="HSV",
children=iaa.Noop(),
name="WithColorspace"),
iaa.WithHueAndSaturation(children=iaa.Noop(),
name="WithHueAndSaturation"),
iaa.MultiplyHueAndSaturation((0.8, 1.2),
name="MultiplyHueAndSaturation"),
iaa.MultiplyHue((-1.0, 1.0), name="MultiplyHue"),
iaa.MultiplySaturation((0.8, 1.2), name="MultiplySaturation"),
iaa.AddToHueAndSaturation((-10, 10), name="AddToHueAndSaturation"),
iaa.AddToHue((-10, 10), name="AddToHue"),
iaa.AddToSaturation((-10, 10), name="AddToSaturation"),
iaa.ChangeColorspace(to_colorspace="HSV", name="ChangeColorspace"),
iaa.Grayscale((0.01, 0.99), name="Grayscale"),
iaa.KMeansColorQuantization((2, 16), name="KMeansColorQuantization"),
iaa.UniformColorQuantization((2, 16), name="UniformColorQuantization")
]
augmenters_contrast = [
iaa.GammaContrast(gamma=(0.5, 2.0), name="GammaContrast"),
iaa.SigmoidContrast(gain=(5, 20),
cutoff=(0.25, 0.75),
name="SigmoidContrast"),
iaa.LogContrast(gain=(0.7, 1.0), name="LogContrast"),
iaa.LinearContrast((0.5, 1.5), name="LinearContrast"),
iaa.AllChannelsCLAHE(clip_limit=(2, 10),
tile_grid_size_px=(3, 11),
name="AllChannelsCLAHE"),
iaa.CLAHE(clip_limit=(2, 10),
tile_grid_size_px=(3, 11),
to_colorspace="HSV",
name="CLAHE"),
iaa.AllChannelsHistogramEqualization(
name="AllChannelsHistogramEqualization"),
iaa.HistogramEqualization(to_colorspace="HSV",
name="HistogramEqualization"),
]
augmenters_convolutional = [
iaa.Convolve(np.float32([[0, 0, 0], [0, 1, 0], [0, 0, 0]]),
name="Convolve_3x3"),
iaa.Sharpen(alpha=(0.01, 0.99), lightness=(0.5, 2), name="Sharpen"),
iaa.Emboss(alpha=(0.01, 0.99), strength=(0, 2), name="Emboss"),
iaa.EdgeDetect(alpha=(0.01, 0.99), name="EdgeDetect"),
iaa.DirectedEdgeDetect(alpha=(0.01, 0.99), name="DirectedEdgeDetect")
]
augmenters_edges = [iaa.Canny(alpha=(0.01, 0.99), name="Canny")]
augmenters_flip = [
iaa.Fliplr(1.0, name="Fliplr"),
iaa.Flipud(1.0, name="Flipud")
]
augmenters_geometric = [
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=0,
mode="constant",
cval=(0, 255),
name="Affine_order_0_constant"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=1,
mode="constant",
cval=(0, 255),
name="Affine_order_1_constant"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=3,
mode="constant",
cval=(0, 255),
name="Affine_order_3_constant"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=1,
mode="edge",
cval=(0, 255),
name="Affine_order_1_edge"),
iaa.Affine(scale=(0.9, 1.1),
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-10, 10),
shear=(-10, 10),
order=1,
mode="constant",
cval=(0, 255),
backend="skimage",
name="Affine_order_1_constant_skimage"),
# TODO AffineCv2
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=4,
nb_cols=4,
order=1,
mode="constant",
name="PiecewiseAffine_4x4_order_1_constant"),
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=4,
nb_cols=4,
order=0,
mode="constant",
name="PiecewiseAffine_4x4_order_0_constant"),
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=4,
nb_cols=4,
order=1,
mode="edge",
name="PiecewiseAffine_4x4_order_1_edge"),
iaa.PiecewiseAffine(scale=(0.01, 0.05),
nb_rows=8,
nb_cols=8,
order=1,
mode="constant",
name="PiecewiseAffine_8x8_order_1_constant"),
iaa.PerspectiveTransform(scale=(0.01, 0.05),
keep_size=False,
name="PerspectiveTransform"),
iaa.PerspectiveTransform(scale=(0.01, 0.05),
keep_size=True,
name="PerspectiveTransform_keep_size"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=0,
mode="constant",
cval=0,
name="ElasticTransformation_order_0_constant"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=1,
mode="constant",
cval=0,
name="ElasticTransformation_order_1_constant"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=1,
mode="nearest",
cval=0,
name="ElasticTransformation_order_1_nearest"),
iaa.ElasticTransformation(
alpha=(1, 10),
sigma=(0.5, 1.5),
order=1,
mode="reflect",
cval=0,
name="ElasticTransformation_order_1_reflect"),
iaa.Rot90((1, 3), keep_size=False, name="Rot90"),
iaa.Rot90((1, 3), keep_size=True, name="Rot90_keep_size")
]
augmenters_pooling = [
iaa.AveragePooling(kernel_size=(1, 16),
keep_size=False,
name="AveragePooling"),
iaa.AveragePooling(kernel_size=(1, 16),
keep_size=True,
name="AveragePooling_keep_size"),
iaa.MaxPooling(kernel_size=(1, 16), keep_size=False,
name="MaxPooling"),
iaa.MaxPooling(kernel_size=(1, 16),
keep_size=True,
name="MaxPooling_keep_size"),
iaa.MinPooling(kernel_size=(1, 16), keep_size=False,
name="MinPooling"),
iaa.MinPooling(kernel_size=(1, 16),
keep_size=True,
name="MinPooling_keep_size"),
iaa.MedianPooling(kernel_size=(1, 16),
keep_size=False,
name="MedianPooling"),
iaa.MedianPooling(kernel_size=(1, 16),
keep_size=True,
name="MedianPooling_keep_size")
]
augmenters_segmentation = [
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=64,
interpolation="cubic",
name="Superpixels_max_size_64_cubic"),
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=64,
interpolation="linear",
name="Superpixels_max_size_64_linear"),
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=128,
interpolation="linear",
name="Superpixels_max_size_128_linear"),
iaa.Superpixels(p_replace=(0.05, 1.0),
n_segments=(10, 100),
max_size=224,
interpolation="linear",
name="Superpixels_max_size_224_linear"),
iaa.UniformVoronoi(n_points=(250, 1000), name="UniformVoronoi"),
iaa.RegularGridVoronoi(n_rows=(16, 31),
n_cols=(16, 31),
name="RegularGridVoronoi"),
iaa.RelativeRegularGridVoronoi(n_rows_frac=(0.07, 0.14),
n_cols_frac=(0.07, 0.14),
name="RelativeRegularGridVoronoi"),
]
augmenters_size = [
iaa.Resize((0.8, 1.2), interpolation="nearest", name="Resize_nearest"),
iaa.Resize((0.8, 1.2), interpolation="linear", name="Resize_linear"),
iaa.Resize((0.8, 1.2), interpolation="cubic", name="Resize_cubic"),
iaa.CropAndPad(percent=(-0.2, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
keep_size=False,
name="CropAndPad"),
iaa.CropAndPad(percent=(-0.2, 0.2),
pad_mode="edge",
pad_cval=(0, 255),
keep_size=False,
name="CropAndPad_edge"),
iaa.CropAndPad(percent=(-0.2, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
name="CropAndPad_keep_size"),
iaa.Pad(percent=(0.05, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
keep_size=False,
name="Pad"),
iaa.Pad(percent=(0.05, 0.2),
pad_mode="edge",
pad_cval=(0, 255),
keep_size=False,
name="Pad_edge"),
iaa.Pad(percent=(0.05, 0.2),
pad_mode="constant",
pad_cval=(0, 255),
name="Pad_keep_size"),
iaa.Crop(percent=(0.05, 0.2), keep_size=False, name="Crop"),
iaa.Crop(percent=(0.05, 0.2), name="Crop_keep_size"),
iaa.PadToFixedSize(width=width + 10,
height=height + 10,
pad_mode="constant",
pad_cval=(0, 255),
name="PadToFixedSize"),
iaa.CropToFixedSize(width=width - 10,
height=height - 10,
name="CropToFixedSize"),
iaa.KeepSizeByResize(iaa.CropToFixedSize(height=height - 10,
width=width - 10),
interpolation="nearest",
name="KeepSizeByResize_CropToFixedSize_nearest"),
iaa.KeepSizeByResize(iaa.CropToFixedSize(height=height - 10,
width=width - 10),
interpolation="linear",
name="KeepSizeByResize_CropToFixedSize_linear"),
iaa.KeepSizeByResize(iaa.CropToFixedSize(height=height - 10,
width=width - 10),
interpolation="cubic",
name="KeepSizeByResize_CropToFixedSize_cubic"),
]
augmenters_weather = [
iaa.FastSnowyLandscape(lightness_threshold=(100, 255),
lightness_multiplier=(1.0, 4.0),
name="FastSnowyLandscape"),
iaa.Clouds(name="Clouds"),
iaa.Fog(name="Fog"),
iaa.CloudLayer(intensity_mean=(196, 255),
intensity_freq_exponent=(-2.5, -2.0),
intensity_coarse_scale=10,
alpha_min=0,
alpha_multiplier=(0.25, 0.75),
alpha_size_px_max=(2, 8),
alpha_freq_exponent=(-2.5, -2.0),
sparsity=(0.8, 1.0),
density_multiplier=(0.5, 1.0),
name="CloudLayer"),
iaa.Snowflakes(name="Snowflakes"),
iaa.SnowflakesLayer(density=(0.005, 0.075),
density_uniformity=(0.3, 0.9),
flake_size=(0.2, 0.7),
flake_size_uniformity=(0.4, 0.8),
angle=(-30, 30),
speed=(0.007, 0.03),
blur_sigma_fraction=(0.0001, 0.001),
name="SnowflakesLayer")
]
augmenters = (augmenters_meta + augmenters_arithmetic + augmenters_blend +
augmenters_blur + augmenters_color + augmenters_contrast +
augmenters_convolutional + augmenters_edges +
augmenters_flip + augmenters_geometric + augmenters_pooling +
augmenters_segmentation + augmenters_size +
augmenters_weather)
if only_augmenters is not None:
augmenters_reduced = []
for augmenter in augmenters:
if any([
re.search(pattern, augmenter.name)
for pattern in only_augmenters
]):
augmenters_reduced.append(augmenter)
augmenters = augmenters_reduced
return augmenters
def test_determinism():
reseed()
images = [
ia.quokka(size=(128, 128)),
ia.quokka(size=(64, 64)),
ia.imresize_single_image(skimage.data.astronaut(), (128, 256))
]
images.extend([ia.quokka(size=(16, 16))] * 20)
keypoints = [
ia.KeypointsOnImage([
ia.Keypoint(x=20, y=10),
ia.Keypoint(x=5, y=5),
ia.Keypoint(x=10, y=43)
],
shape=(50, 60, 3))
] * 20
augs = [
iaa.Sequential([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.SomeOf(1, [iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.OneOf([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.Sometimes(0.5, iaa.Fliplr(1.0)),
iaa.WithColorspace("HSV", children=iaa.Add((-50, 50))),
iaa.Resize((0.5, 0.9)),
iaa.CropAndPad(px=(-50, 50)),
iaa.Pad(px=(1, 50)),
iaa.Crop(px=(1, 50)),
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Superpixels(p_replace=(0.25, 1.0), n_segments=(16, 128)),
iaa.Grayscale(alpha=(0.1, 1.0)),
iaa.GaussianBlur((0.1, 3.0)),
iaa.AverageBlur((3, 11)),
iaa.MedianBlur((3, 11)),
iaa.Sharpen(alpha=(0.1, 1.0), lightness=(0.8, 1.2)),
iaa.Emboss(alpha=(0.1, 1.0), strength=(0.8, 1.2)),
iaa.EdgeDetect(alpha=(0.1, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.1, 1.0), direction=(0.0, 1.0)),
iaa.Add((-50, 50)),
iaa.AddElementwise((-50, 50)),
iaa.AdditiveGaussianNoise(scale=(0.1, 1.0)),
iaa.Multiply((0.6, 1.4)),
iaa.MultiplyElementwise((0.6, 1.4)),
iaa.Dropout((0.3, 0.5)),
iaa.CoarseDropout((0.3, 0.5), size_percent=(0.05, 0.2)),
iaa.Invert(0.5),
iaa.Affine(scale=(0.7, 1.3),
translate_percent=(-0.1, 0.1),
rotate=(-20, 20),
shear=(-20, 20),
order=ia.ALL,
mode=ia.ALL,
cval=(0, 255)),
iaa.PiecewiseAffine(scale=(0.1, 0.3)),
iaa.ElasticTransformation(alpha=0.5)
]
augs_affect_geometry = [
iaa.Sequential([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.SomeOf(1, [iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.OneOf([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.Sometimes(0.5, iaa.Fliplr(1.0)),
iaa.Resize((0.5, 0.9)),
iaa.CropAndPad(px=(-50, 50)),
iaa.Pad(px=(1, 50)),
iaa.Crop(px=(1, 50)),
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Affine(scale=(0.7, 1.3),
translate_percent=(-0.1, 0.1),
rotate=(-20, 20),
shear=(-20, 20),
order=ia.ALL,
mode=ia.ALL,
cval=(0, 255)),
iaa.PiecewiseAffine(scale=(0.1, 0.3)),
iaa.ElasticTransformation(alpha=(5, 100), sigma=(3, 5))
]
for aug in augs:
aug_det = aug.to_deterministic()
images_aug1 = aug_det.augment_images(images)
images_aug2 = aug_det.augment_images(images)
aug_det = aug.to_deterministic()
images_aug3 = aug_det.augment_images(images)
images_aug4 = aug_det.augment_images(images)
assert array_equal_lists(images_aug1, images_aug2), \
"Images (1, 2) expected to be identical for %s" % (aug.name,)
assert array_equal_lists(images_aug3, images_aug4), \
"Images (3, 4) expected to be identical for %s" % (aug.name,)
assert not array_equal_lists(images_aug1, images_aug3), \
"Images (1, 3) expected to be different for %s" % (aug.name,)
for aug in augs_affect_geometry:
aug_det = aug.to_deterministic()
kps_aug1 = aug_det.augment_keypoints(keypoints)
kps_aug2 = aug_det.augment_keypoints(keypoints)
aug_det = aug.to_deterministic()
kps_aug3 = aug_det.augment_keypoints(keypoints)
kps_aug4 = aug_det.augment_keypoints(keypoints)
assert keypoints_equal(kps_aug1, kps_aug2), \
"Keypoints (1, 2) expected to be identical for %s" % (aug.name,)
assert keypoints_equal(kps_aug3, kps_aug4), \
"Keypoints (3, 4) expected to be identical for %s" % (aug.name,)
assert not keypoints_equal(kps_aug1, kps_aug3), \
"Keypoints (1, 3) expected to be different for %s" % (aug.name,)
def read_video(file: str,
src_folder: str,
dst_folder: str,
verbose: bool = False):
seq = iaa.Sequential(
[iaa.CoarseDropout((0.0, 0.10), size_percent=(0.02, 0.25))])
# Create a VideoCapture object and read from input file
# If the input is the camera, pass 0 instead of the video file name
src_path = os.path.join(src_folder, file)
dst_path = dst_folder
cap = cv2.VideoCapture(src_path)
# Check if camera opened successfully
if (cap.isOpened() == False):
print("Error opening video stream or file")
# make destination folder
if not os.path.exists(dst_path):
os.makedirs(dst_path)
# frame index
frame_idx = 0
# Read until video is completed
while (cap.isOpened()):
# Capture frame-by-frame
ret, frame = cap.read()
if ret == True:
frame = frame[:320, ...]
# Display the resulting frame
if verbose:
cv2.imshow('Frame', frame)
# save frame image
img_path = os.path.join(dst_path, "imgs",
file[:-6] + "." + str(frame_idx) + ".png")
img = pil.fromarray(frame[..., ::-1])
resized_img = img.resize((256, 128))
resized_img.save(img_path, 'png')
# predict depth and save it
resized_img = img.resize((512, 256))
resized_img = np.asarray(resized_img)
depth = get_depth_map(resized_img)
resized_depth = cv2.resize(depth, dsize=(256, 128))
depth_path = os.path.join(
dst_path, "depths", file[:-6] + "." + str(frame_idx) + ".pkl")
with open(depth_path, "wb") as fout:
pkl.dump(resized_depth, fout)
#save camera intrinsic
camera_intrinsic = get_camera_intrinsic()
camera_intrinsic_path = os.path.join(
dst_path, "intrinsics",
file[:-6] + "." + str(frame_idx) + ".pkl")
with open(camera_intrinsic_path, "wb") as fout:
pkl.dump(camera_intrinsic, fout)
# save camera extrinsic
camera_extrinsic = get_camera_extrinsic()
camera_extrinsic_path = os.path.join(
dst_path, "extrinsics",
file[:-6] + "." + str(frame_idx) + ".pkl")
with open(camera_extrinsic_path, "wb") as fout:
pkl.dump(camera_extrinsic, fout)
# generate mask
timg = torch.tensor(resized_img.transpose(
2, 0, 1)).unsqueeze(0).double()
tdepth = torch.tensor(resized_depth).unsqueeze(0).double()
tintrinsic = torch.tensor(camera_intrinsic).unsqueeze(0).double()
textrinsic = torch.tensor(camera_extrinsic).unsqueeze(0).double()
# projection
tprojected_img, tvalid_points = random_transformation(
timg, tintrinsic, textrinsic, tdepth)
# mask
mask = 255 * tvalid_points.to(torch.uint8).squeeze(0).numpy()
mask = seq(images=mask)
mask_path = os.path.join(dst_path, "masks",
file[:-6] + "." + str(frame_idx) + ".png")
cv2.imwrite(mask_path, mask)
# increment number of frame
frame_idx += 1
# Press Q on keyboard to exit
if cv2.waitKey(25) & 0xFF == ord('q'):
break
# Break the loop
else:
break
# When everything done, release the video capture object
cap.release()
# Closes all the frames
cv2.destroyAllWindows()
def test_dtype_preservation():
reseed()
size = (4, 16, 16, 3)
images = [
np.random.uniform(0, 255, size).astype(np.uint8),
np.random.uniform(0, 65535, size).astype(np.uint16),
np.random.uniform(0, 4294967295, size).astype(np.uint32),
np.random.uniform(-128, 127, size).astype(np.int16),
np.random.uniform(-32768, 32767, size).astype(np.int32),
np.random.uniform(0.0, 1.0, size).astype(np.float32),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float16),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float32),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float64)
]
default_dtypes = set([arr.dtype for arr in images])
# Some dtypes are here removed per augmenter, because the respective
# augmenter does not support them. This test currently only checks whether
# dtypes are preserved from in- to output for all dtypes that are supported
# per augmenter.
# dtypes are here removed via list comprehension instead of
# `default_dtypes - set([dtype])`, because the latter one simply never
# removed the dtype(s) for some reason
def _not_dts(dts):
return [dt for dt in default_dtypes if dt not in dts]
augs = [
(iaa.Add((-5, 5),
name="Add"), _not_dts([np.uint32, np.int32, np.float64])),
(iaa.AddElementwise((-5, 5), name="AddElementwise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.AdditiveGaussianNoise(0.01 * 255, name="AdditiveGaussianNoise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Multiply((0.95, 1.05), name="Multiply"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Dropout(0.01, name="Dropout"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.CoarseDropout(0.01, size_px=6, name="CoarseDropout"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Invert(0.01, per_channel=True, name="Invert"), default_dtypes),
(iaa.GaussianBlur(sigma=(0.95, 1.05),
name="GaussianBlur"), _not_dts([np.float16])),
(iaa.AverageBlur((3, 5), name="AverageBlur"),
_not_dts([np.uint32, np.int32, np.float16])),
(iaa.MedianBlur((3, 5), name="MedianBlur"),
_not_dts([np.uint32, np.int32, np.float16, np.float64])),
(iaa.BilateralBlur((3, 5), name="BilateralBlur"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float16, np.float64
])),
(iaa.Sharpen((0.0, 0.1), lightness=(1.0, 1.2), name="Sharpen"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.Emboss(alpha=(0.0, 0.1), strength=(0.5, 1.5), name="Emboss"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.EdgeDetect(alpha=(0.0, 0.1), name="EdgeDetect"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.DirectedEdgeDetect(alpha=(0.0, 0.1),
direction=0,
name="DirectedEdgeDetect"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.Fliplr(0.5, name="Fliplr"), default_dtypes),
(iaa.Flipud(0.5, name="Flipud"), default_dtypes),
(iaa.Affine(translate_px=(-5, 5), name="Affine-translate-px"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(translate_percent=(-0.05, 0.05),
name="Affine-translate-percent"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(rotate=(-20, 20),
name="Affine-rotate"), _not_dts([np.uint32, np.int32])),
(iaa.Affine(shear=(-20, 20),
name="Affine-shear"), _not_dts([np.uint32, np.int32])),
(iaa.Affine(scale=(0.9, 1.1),
name="Affine-scale"), _not_dts([np.uint32, np.int32])),
(iaa.PiecewiseAffine(scale=(0.001, 0.005),
name="PiecewiseAffine"), default_dtypes),
(iaa.ElasticTransformation(alpha=(0.1, 0.2),
sigma=(0.1, 0.2),
name="ElasticTransformation"),
_not_dts([np.float16])),
(iaa.Sequential([iaa.Noop(), iaa.Noop()],
name="SequentialNoop"), default_dtypes),
(iaa.SomeOf(1, [iaa.Noop(), iaa.Noop()],
name="SomeOfNoop"), default_dtypes),
(iaa.OneOf([iaa.Noop(), iaa.Noop()],
name="OneOfNoop"), default_dtypes),
(iaa.Sometimes(0.5, iaa.Noop(), name="SometimesNoop"), default_dtypes),
(iaa.Sequential([iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))],
name="Sequential"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.SomeOf(1, [iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))],
name="SomeOf"), _not_dts([np.uint32, np.int32,
np.float64])),
(iaa.OneOf([iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))],
name="OneOf"), _not_dts([np.uint32, np.int32, np.float64])),
(iaa.Sometimes(0.5, iaa.Add((-5, 5)), name="Sometimes"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Noop(name="Noop"), default_dtypes),
(iaa.Alpha((0.0, 0.1), iaa.Noop(), name="AlphaNoop"), default_dtypes),
(iaa.AlphaElementwise((0.0, 0.1),
iaa.Noop(),
name="AlphaElementwiseNoop"), default_dtypes),
(iaa.SimplexNoiseAlpha(iaa.Noop(),
name="SimplexNoiseAlphaNoop"), default_dtypes),
(iaa.FrequencyNoiseAlpha(exponent=(-2, 2),
first=iaa.Noop(),
name="SimplexNoiseAlphaNoop"),
default_dtypes),
(iaa.Alpha((0.0, 0.1), iaa.Add(10),
name="Alpha"), _not_dts([np.uint32, np.int32, np.float64])),
(iaa.AlphaElementwise((0.0, 0.1), iaa.Add(10),
name="AlphaElementwise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.SimplexNoiseAlpha(iaa.Add(10), name="SimplexNoiseAlpha"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.FrequencyNoiseAlpha(exponent=(-2, 2),
first=iaa.Add(10),
name="SimplexNoiseAlpha"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Superpixels(p_replace=0.01, n_segments=64),
_not_dts([np.float16, np.float32, np.float64])),
(iaa.Resize({
"height": 4,
"width": 4
}, name="Resize"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
])),
(iaa.CropAndPad(px=(-10, 10), name="CropAndPad"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
])),
(iaa.Pad(px=(0, 10), name="Pad"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
])),
(iaa.Crop(px=(0, 10), name="Crop"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
]))
]
for (aug, allowed_dtypes) in augs:
for images_i in images:
if images_i.dtype in allowed_dtypes:
images_aug = aug.augment_images(images_i)
assert images_aug.dtype == images_i.dtype
def agum_data(raw_path,agum_path):
crop=iaa.CropAndPad(percent=(-0.15, 0.15),sample_independently=False,keep_size=False)
drop=iaa.CoarseDropout(0.02, size_percent=0.15, per_channel=0.5)
gauss = iaa.GaussianBlur(sigma=(0.0, 3.0))
agum = iaa.SomeOf(2, [crop,drop,gauss])
agum_template(raw_path,agum_path,agum,n_iters=4)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(lr=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False),
metrics=['accuracy'])
# Initiate the train and test generators with data Augmentation
sometimes = lambda aug: iaa.Sometimes(0.6, aug)
seq = iaa.Sequential([
iaa.GaussianBlur(sigma=(0, 1.0)),
iaa.Sharpen(alpha=1, lightness=0),
iaa.CoarseDropout(p=0.1, size_percent=0.15),
sometimes(
iaa.Affine(scale={
'x': (0.8, 1.2),
'y': (0.8, 1.2)
},
translate_percent={
'x': (-0.2, 0.2),
'y': (-0.2, 0.2)
},
rotate=(-30, 30),
shear=(-16, 16)))
])
train_datagen = ImageDataGenerator(rescale=1. / 255,
preprocessing_function=seq.augment_image,
def main():
parser = argparse.ArgumentParser(description="Check augmenters visually.")
parser.add_argument(
'--only',
default=None,
help=
"If this is set, then only the results of an augmenter with this name will be shown. Optionally, comma-separated list.",
required=False)
args = parser.parse_args()
images = [
ia.quokka_square(size=(128, 128)),
ia.imresize_single_image(data.astronaut(), (128, 128))
]
keypoints = [
ia.KeypointsOnImage([
ia.Keypoint(x=50, y=40),
ia.Keypoint(x=70, y=38),
ia.Keypoint(x=62, y=52)
],
shape=images[0].shape),
ia.KeypointsOnImage([
ia.Keypoint(x=55, y=32),
ia.Keypoint(x=42, y=95),
ia.Keypoint(x=75, y=89)
],
shape=images[1].shape)
]
bounding_boxes = [
ia.BoundingBoxesOnImage([
ia.BoundingBox(x1=10, y1=10, x2=20, y2=20),
ia.BoundingBox(x1=40, y1=50, x2=70, y2=60)
],
shape=images[0].shape),
ia.BoundingBoxesOnImage([
ia.BoundingBox(x1=10, y1=10, x2=20, y2=20),
ia.BoundingBox(x1=40, y1=50, x2=70, y2=60)
],
shape=images[1].shape)
]
# missing: InColorspace, Lambda, AssertLambda, AssertShape, Convolve
augmenters = [
iaa.Sequential([
iaa.CoarseDropout(p=0.5, size_percent=0.05),
iaa.AdditiveGaussianNoise(scale=0.1 * 255),
iaa.Crop(percent=0.1)
],
name="Sequential"),
iaa.SomeOf(2,
children=[
iaa.CoarseDropout(p=0.5, size_percent=0.05),
iaa.AdditiveGaussianNoise(scale=0.1 * 255),
iaa.Crop(percent=0.1)
],
name="SomeOf"),
iaa.OneOf(children=[
iaa.CoarseDropout(p=0.5, size_percent=0.05),
iaa.AdditiveGaussianNoise(scale=0.1 * 255),
iaa.Crop(percent=0.1)
],
name="OneOf"),
iaa.Sometimes(0.5,
iaa.AdditiveGaussianNoise(scale=0.1 * 255),
name="Sometimes"),
iaa.WithColorspace("HSV",
children=[iaa.Add(20)],
name="WithColorspace"),
iaa.WithChannels([0], children=[iaa.Add(20)], name="WithChannels"),
iaa.AddToHueAndSaturation((-20, 20),
per_channel=True,
name="AddToHueAndSaturation"),
iaa.Noop(name="Noop"),
iaa.Scale({
"width": 64,
"height": 64
}, name="Scale"),
iaa.CropAndPad(px=(-8, 8), name="CropAndPad-px"),
iaa.Pad(px=(0, 8), name="Pad-px"),
iaa.Crop(px=(0, 8), name="Crop-px"),
iaa.Crop(percent=(0, 0.1), name="Crop-percent"),
iaa.Fliplr(0.5, name="Fliplr"),
iaa.Flipud(0.5, name="Flipud"),
iaa.Superpixels(p_replace=0.75, n_segments=50, name="Superpixels"),
iaa.Grayscale(0.5, name="Grayscale0.5"),
iaa.Grayscale(1.0, name="Grayscale1.0"),
iaa.GaussianBlur((0, 3.0), name="GaussianBlur"),
iaa.AverageBlur(k=(3, 11), name="AverageBlur"),
iaa.MedianBlur(k=(3, 11), name="MedianBlur"),
iaa.BilateralBlur(d=10, name="BilateralBlur"),
iaa.Sharpen(alpha=(0.1, 1.0), lightness=(0, 2.0), name="Sharpen"),
iaa.Emboss(alpha=(0.1, 1.0), strength=(0, 2.0), name="Emboss"),
iaa.EdgeDetect(alpha=(0.1, 1.0), name="EdgeDetect"),
iaa.DirectedEdgeDetect(alpha=(0.1, 1.0),
direction=(0, 1.0),
name="DirectedEdgeDetect"),
iaa.Add((-50, 50), name="Add"),
iaa.Add((-50, 50), per_channel=True, name="AddPerChannel"),
iaa.AddElementwise((-50, 50), name="AddElementwise"),
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, 0.1 * 255),
name="AdditiveGaussianNoise"),
iaa.Multiply((0.5, 1.5), name="Multiply"),
iaa.Multiply((0.5, 1.5), per_channel=True, name="MultiplyPerChannel"),
iaa.MultiplyElementwise((0.5, 1.5), name="MultiplyElementwise"),
iaa.Dropout((0.0, 0.1), name="Dropout"),
iaa.CoarseDropout(p=0.05,
size_percent=(0.05, 0.5),
name="CoarseDropout"),
iaa.Invert(p=0.5, name="Invert"),
iaa.Invert(p=0.5, per_channel=True, name="InvertPerChannel"),
iaa.ContrastNormalization(alpha=(0.5, 2.0),
name="ContrastNormalization"),
iaa.SaltAndPepper(p=0.05, name="SaltAndPepper"),
iaa.Salt(p=0.05, name="Salt"),
iaa.Pepper(p=0.05, name="Pepper"),
iaa.CoarseSaltAndPepper(p=0.05,
size_percent=(0.01, 0.1),
name="CoarseSaltAndPepper"),
iaa.CoarseSalt(p=0.05, size_percent=(0.01, 0.1), name="CoarseSalt"),
iaa.CoarsePepper(p=0.05, size_percent=(0.01, 0.1),
name="CoarsePepper"),
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_px={
"x": (-16, 16),
"y": (-16, 16)
},
rotate=(-45, 45),
shear=(-16, 16),
order=ia.ALL,
cval=(0, 255),
mode=ia.ALL,
name="Affine"),
iaa.PiecewiseAffine(scale=0.03,
nb_rows=(2, 6),
nb_cols=(2, 6),
name="PiecewiseAffine"),
iaa.PerspectiveTransform(scale=0.1, name="PerspectiveTransform"),
iaa.ElasticTransformation(alpha=(0.5, 8.0),
sigma=1.0,
name="ElasticTransformation"),
iaa.Alpha(factor=(0.0, 1.0),
first=iaa.Add(100),
second=iaa.Dropout(0.5),
per_channel=False,
name="Alpha"),
iaa.Alpha(factor=(0.0, 1.0),
first=iaa.Add(100),
second=iaa.Dropout(0.5),
per_channel=True,
name="AlphaPerChannel"),
iaa.Alpha(factor=(0.0, 1.0),
first=iaa.Affine(rotate=(-45, 45)),
per_channel=True,
name="AlphaAffine"),
iaa.AlphaElementwise(factor=(0.0, 1.0),
first=iaa.Add(50),
second=iaa.ContrastNormalization(2.0),
per_channel=False,
name="AlphaElementwise"),
iaa.AlphaElementwise(factor=(0.0, 1.0),
first=iaa.Add(50),
second=iaa.ContrastNormalization(2.0),
per_channel=True,
name="AlphaElementwisePerChannel"),
iaa.AlphaElementwise(factor=(0.0, 1.0),
first=iaa.Affine(rotate=(-45, 45)),
per_channel=True,
name="AlphaElementwiseAffine"),
iaa.SimplexNoiseAlpha(
#first=iaa.GaussianBlur((1.0, 3.0)),
#first=iaa.MedianBlur((3, 7)),
first=iaa.EdgeDetect(1.0),
#first=iaa.Affine(rotate=-45), #(-45, 45)),
per_channel=False,
name="SimplexNoiseAlpha"),
iaa.FrequencyNoiseAlpha(
#first=iaa.GaussianBlur((1.0, 3.0)),
#first=iaa.MedianBlur((3, 7)),
first=iaa.EdgeDetect(1.0),
#first=iaa.Affine(rotate=-45), #(-45, 45)),
per_channel=False,
name="FrequencyNoiseAlpha")
]
augmenters.append(
iaa.Sequential([iaa.Sometimes(0.2, aug.copy()) for aug in augmenters],
name="Sequential"))
augmenters.append(
iaa.Sometimes(0.5, [aug.copy() for aug in augmenters],
name="Sometimes"))
for augmenter in augmenters:
if args.only is None or augmenter.name in [
v.strip() for v in args.only.split(",")
]:
print("Augmenter: %s" % (augmenter.name, ))
grid = []
for image, kps, bbs in zip(images, keypoints, bounding_boxes):
aug_det = augmenter.to_deterministic()
imgs_aug = aug_det.augment_images(
np.tile(image[np.newaxis, ...], (16, 1, 1, 1)))
kps_aug = aug_det.augment_keypoints([kps] * 16)
bbs_aug = aug_det.augment_bounding_boxes([bbs] * 16)
imgs_aug_drawn = [
kps_aug_one.draw_on_image(img_aug)
for img_aug, kps_aug_one in zip(imgs_aug, kps_aug)
]
imgs_aug_drawn = [
bbs_aug_one.draw_on_image(img_aug)
for img_aug, bbs_aug_one in zip(imgs_aug_drawn, bbs_aug)
]
grid.append(np.hstack(imgs_aug_drawn))
ia.imshow(np.vstack(grid))
def aug_artifact_error(batch, min_drop, max_drop, size):
aug = iaa.CoarseDropout((min_drop, max_drop), size_percent=size)
return aug(images=batch)
def __init__(self, path='/'):
configMain.__init__(self)
st = lambda aug: iaa.Sometimes(0.2, aug)
oc = lambda aug: iaa.Sometimes(0.1, aug)
rl = lambda aug: iaa.Sometimes(0.05, aug)
self.augment = [
iaa.Sequential(
[
rl(iaa.GaussianBlur(
(0,
1.3))), # blur images with a sigma between 0 and 1.5
rl(
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05),
per_channel=0.5)), # add gaussian noise to images
rl(iaa.Dropout((0.0, 0.10), per_channel=0.5)
), # randomly remove up to X% of the pixels
oc(
iaa.Add((-20, 20), per_channel=0.5)
), # change brightness of images (by -X to Y of original value)
st(iaa.Multiply(
(0.25, 2.5), per_channel=0.2
)), # change brightness of images (X-Y% of original value)
rl(iaa.ContrastNormalization(
(0.5, 1.5),
per_channel=0.5)), # improve or worsen the contrast
rl(iaa.Grayscale((0.0, 1))), # put grayscale
],
random_order=True # do all of the above in random order
),
iaa.Sequential(
[ # AUGMENTATION Labels
rl(iaa.Dropout(
(0.0,
0.03))), # randomly remove up to X% of the pixels
rl(iaa.CoarseDropout((0.0, 0.03), size_percent=(
0.2, 0.3))), # randomly remove up to X% of the pixels
],
random_order=True # do all of the above in random order
)
]
# self.augment = [None]
# there are files with data, 200 images each, and here we select which ones to use
self.dataset_name = 'Carla'
train_path = os.path.join(path, 'RC28_wpz_M_DR')
val_path = os.path.join(path, 'RC28_wpz_M_DR')
print(train_path)
self.train_db_path = [
os.path.join(train_path, f)
for f in glob.glob1(train_path, "data_*.h5")
]
self.val_db_path = [
os.path.join(val_path, f)
for f in glob.glob1(val_path, "data_*.h5")
]
# Speed Divide Factor
# TODO: FOr now is hardcooded, but eventually we should be able to calculate this from data at the loading time.
self.speed_factor = 40.0 # In KM/H FOR GTA it should be maximun 30.0
# The division is made by three diferent data kinds
# in every mini-batch there will be equal number of samples with labels from each group
# e.g. for [[0,1],[2]] there will be 50% samples with labels 0 and 1, and 50% samples with label 2
self.labels_per_division = [[0, 2, 5], [3], [4]]
self.dataset_names = ['targets']
self.queue_capacity = 20 * self.batch_size
def __init__(self):
configMain.__init__(self)
st = lambda aug: iaa.Sometimes(0.4, aug)
oc = lambda aug: iaa.Sometimes(0.3, aug)
rl = lambda aug: iaa.Sometimes(0.09, aug)
self.augment = iaa.Sequential(
[
rl(iaa.GaussianBlur(
(0, 1.5))), # blur images with a sigma between 0 and 1.5
rl(
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05),
per_channel=0.5)), # add gaussian noise to images
oc(iaa.Dropout((0.0, 0.10), per_channel=0.5)
), # randomly remove up to X% of the pixels
oc(
iaa.CoarseDropout(
(0.0, 0.10), size_percent=(0.08, 0.2), per_channel=0.5)
), # randomly remove up to X% of the pixels
oc(
iaa.Add((-40, 40), per_channel=0.5)
), # change brightness of images (by -X to Y of original value)
st(iaa.Multiply((0.10, 2.5), per_channel=0.2)
), # change brightness of images (X-Y% of original value)
rl(iaa.ContrastNormalization(
(0.5, 1.5),
per_channel=0.5)), # improve or worsen the contrast
rl(iaa.Grayscale((0.0, 1))), # put grayscale
],
random_order=True # do all of the above in random order
)
self.augment_labels = False
# there are files with data, 200 images each, and here we select which ones to use
#5self.dataset_name = 'Carla'
#with open(os.path.join(self.save_data_stats, 'path'),'r') as f:
# path = f.read().strip()
path = '../ElektraData4'
train_path = os.path.join(path, 'SeqTrain')
val_path = os.path.join(path, 'SeqVal')
print train_path, val_path
self.train_db_path = [
os.path.join(train_path, f)
for f in glob.glob1(train_path, "data_*.h5")
]
self.val_db_path = [
os.path.join(val_path, f)
for f in glob.glob1(val_path, "data_*.h5")
]
# When using data with noise, remove the recording during the first half of the noise impulse
# TODO Felipe: change to noise percentage.
self.remove_noise = False
# Speed Divide Factor
#TODO: FOr now is hardcooded, but eventually we should be able to calculate this from data at the loading time.
self.speed_factor = 1.0 # In KM/H FOR GTA it should be maximun 30.0
# The division is made by three diferent data kinds
# in every mini-batch there will be equal number of samples with labels from each group
# e.g. for [[0,1],[2]] there will be 50% samples with labels 0 and 1, and 50% samples with label 2
self.labels_per_division = [[2], [2], [2]]
self.dataset_names = ['targets']
self.queue_capacity = 20 * self.batch_size
# TODO NOT IMPLEMENTED Felipe: True/False switches to turn data balancing on or off
self.balances_val = True
self.balances_train = True
self.saturated_factor = True
def get_seq():
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.5), # vertically flip 50% of all images
sometimes(
iaa.Affine(
scale={
"x": (0.9, 1.6),
"y": (0.9, 1.6)
}, #>20 will cut part of img
translate_percent={
"x": (-0.15, 0.15),
"y": (-0.15, 0.15)
}, # >20% will also cut part of img
rotate=(
-10, 10
), # 45/-45° -> works good with scale + translate to prevent cuts
shear=(-5, 5), # shear by -16 to +16 degrees
mode=ia.ALL)),
iaa.SomeOf(
(0, 4),
[
sometimes(
iaa.Superpixels(p_replace=(0.3, 0.7),
n_segments=(10, 100))
), #superpixel-representation --> better basallamina representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 0.2
)), #small blur effects --> better representation
iaa.AverageBlur(k=(1, 3)), # k must be odd
iaa.MedianBlur(k=(1, 3)), #
]),
iaa.Sharpen(
alpha=(0, 1.0),
lightness=(0.9, 1.1)), #cell wall represenation
iaa.Emboss(alpha=(0, 0.8),
strength=(0, 0.5)), #cell wall represenation
#searching for edges or angles --> blobby mask --> better basallamina representation / nuclei
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(
0.2, 0.4)), #detects edges --> cell wall,..
iaa.DirectedEdgeDetect(
alpha=(0.2, 0.4), direction=(0.0, 1.0)
), #direction will make edges from random directions
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.01 * 255),
per_channel=0.2), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.05, 0.2), per_channel=0.2
), #rnd remove 5-30% in small pixels
iaa.CoarseDropout(
(0.05, 0.2),
size_percent=(0.01, 0.02),
per_channel=0.2), # rnd remove 3% in big pixels
]),
iaa.Invert(0.01,
per_channel=True), # invert color channels
iaa.Add(
(-10, 10), per_channel=0.3
), # change brightness of images (by -10 to 10 of original value)
#iaa.AddToHueAndSaturation((-0.1, 0.1)), # change hue and saturation
#
#either change the brightness of the whole image (sometimes per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.9, 1.1), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-1, 0),
first=iaa.Multiply((0.9, 1.1), per_channel=True),
second=iaa.LinearContrast((0.9, 1.1)))
]),
sometimes(
iaa.ElasticTransformation(alpha=(0, 0.2), sigma=0.1)
), #still not sure: move pixels locally around
sometimes(
iaa.PiecewiseAffine(scale=(0.01, 0.03))
), #still not sure:move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
],
random_order=True)
return seq
def build_augmentation_pipeline(self, apply_prob=0.5):
cfg = self.cfg
sometimes = lambda aug: iaa.Sometimes(apply_prob, aug)
pipeline = iaa.Sequential(random_order=False)
pre_resize = cfg.get("pre_resize")
if pre_resize:
width, height = pre_resize
pipeline.add(iaa.Resize({"height": height, "width": width}))
# Add smart, keypoint-aware image cropping
w, h = self.default_crop_size
pipeline.add(iaa.PadToFixedSize(w, h))
pipeline.add(
augmentation.KeypointAwareCropToFixedSize(
w, h, cfg.get("max_shift", 0.4), cfg.get("crop_sampling", "hybrid")
)
)
if cfg.get("fliplr", False):
opt = cfg.get("fliplr", False)
if type(opt) == int:
pipeline.add(sometimes(iaa.Fliplr(opt)))
else:
pipeline.add(sometimes(iaa.Fliplr(0.5)))
if cfg.get("rotation", False):
opt = cfg.get("rotation", False)
if type(opt) == int:
pipeline.add(sometimes(iaa.Affine(rotate=(-opt, opt))))
else:
pipeline.add(sometimes(iaa.Affine(rotate=(-10, 10))))
if cfg.get("hist_eq", False):
pipeline.add(sometimes(iaa.AllChannelsHistogramEqualization()))
if cfg.get("motion_blur", False):
opts = cfg.get("motion_blur", False)
if type(opts) == list:
opts = dict(opts)
pipeline.add(sometimes(iaa.MotionBlur(**opts)))
else:
pipeline.add(sometimes(iaa.MotionBlur(k=7, angle=(-90, 90))))
if cfg.get("covering", False):
pipeline.add(
sometimes(iaa.CoarseDropout((0, 0.02), size_percent=(0.01, 0.05)))
) # , per_channel=0.5)))
if cfg.get("elastic_transform", False):
pipeline.add(sometimes(iaa.ElasticTransformation(sigma=5)))
if cfg.get("gaussian_noise", False):
opt = cfg.get("gaussian_noise", False)
if type(opt) == int or type(opt) == float:
pipeline.add(
sometimes(
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, opt), per_channel=0.5
)
)
)
else:
pipeline.add(
sometimes(
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5
)
)
)
if cfg.get("grayscale", False):
pipeline.add(sometimes(iaa.Grayscale(alpha=(0.5, 1.0))))
def get_aug_param(cfg_value):
if isinstance(cfg_value, dict):
opt = cfg_value
else:
opt = {}
return opt
cfg_cnt = cfg.get("contrast", {})
cfg_cnv = cfg.get("convolution", {})
contrast_aug = ["histeq", "clahe", "gamma", "sigmoid", "log", "linear"]
for aug in contrast_aug:
aug_val = cfg_cnt.get(aug, False)
cfg_cnt[aug] = aug_val
if aug_val:
cfg_cnt[aug + "ratio"] = cfg_cnt.get(aug + "ratio", 0.1)
convolution_aug = ["sharpen", "emboss", "edge"]
for aug in convolution_aug:
aug_val = cfg_cnv.get(aug, False)
cfg_cnv[aug] = aug_val
if aug_val:
cfg_cnv[aug + "ratio"] = cfg_cnv.get(aug + "ratio", 0.1)
if cfg_cnt["histeq"]:
opt = get_aug_param(cfg_cnt["histeq"])
pipeline.add(
iaa.Sometimes(
cfg_cnt["histeqratio"], iaa.AllChannelsHistogramEqualization(**opt)
)
)
if cfg_cnt["clahe"]:
opt = get_aug_param(cfg_cnt["clahe"])
pipeline.add(
iaa.Sometimes(cfg_cnt["claheratio"], iaa.AllChannelsCLAHE(**opt))
)
if cfg_cnt["log"]:
opt = get_aug_param(cfg_cnt["log"])
pipeline.add(iaa.Sometimes(cfg_cnt["logratio"], iaa.LogContrast(**opt)))
if cfg_cnt["linear"]:
opt = get_aug_param(cfg_cnt["linear"])
pipeline.add(
iaa.Sometimes(cfg_cnt["linearratio"], iaa.LinearContrast(**opt))
)
if cfg_cnt["sigmoid"]:
opt = get_aug_param(cfg_cnt["sigmoid"])
pipeline.add(
iaa.Sometimes(cfg_cnt["sigmoidratio"], iaa.SigmoidContrast(**opt))
)
if cfg_cnt["gamma"]:
opt = get_aug_param(cfg_cnt["gamma"])
pipeline.add(iaa.Sometimes(cfg_cnt["gammaratio"], iaa.GammaContrast(**opt)))
if cfg_cnv["sharpen"]:
opt = get_aug_param(cfg_cnv["sharpen"])
pipeline.add(iaa.Sometimes(cfg_cnv["sharpenratio"], iaa.Sharpen(**opt)))
if cfg_cnv["emboss"]:
opt = get_aug_param(cfg_cnv["emboss"])
pipeline.add(iaa.Sometimes(cfg_cnv["embossratio"], iaa.Emboss(**opt)))
if cfg_cnv["edge"]:
opt = get_aug_param(cfg_cnv["edge"])
pipeline.add(iaa.Sometimes(cfg_cnv["edgeratio"], iaa.EdgeDetect(**opt)))
return pipeline
def __init__(self, rgb_mean, randomImg, insize):
sometimes = lambda aug: iaa.Sometimes(0.7, aug)
self.rand_img_dir = randomImg
self.rgb_mean = rgb_mean
self.inp_dim = insize
#
self.randomImgList = glob.glob( randomImg + '*.jpg')
self.aug = iaa.Sequential([
sometimes(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.1, 0.1), "y": (-0.1, 0.1)}, # translate by -20 to +20 percent (per axis)
rotate=(-25, 25), # rotate by -45 to +45 degrees
shear=(-6, 6), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
iaa.OneOf([
iaa.Fliplr(0.5),
iaa.GaussianBlur(
sigma=iap.Uniform(0.0, 1.0)
),
iaa.BlendAlphaSimplexNoise(
foreground=iaa.BlendAlphaSimplexNoise(
foreground=iaa.EdgeDetect(1.0),
background=iaa.LinearContrast((0.1, .8)),
per_channel=True
),
background=iaa.BlendAlphaFrequencyNoise(
exponent=(-.5, -.1),
foreground=iaa.Affine(
rotate=(-10, 10),
translate_px={"x": (-1, 1), "y": (-1, 1)}
),
# background=iaa.AddToHueAndSaturation((-4, 4)),
# per_channel=True
),
per_channel=True,
aggregation_method="max",
sigmoid=False
),
iaa.BlendAlpha(
factor=(0.2, 0.8),
foreground=iaa.Sharpen(1.0, lightness=2),
background=iaa.CoarseDropout(p=0.1, size_px=8)
),
iaa.BlendAlpha(
factor=(0.2, 0.8),
foreground=iaa.Affine(rotate=(-5, 5)),
per_channel=True
),
iaa.MotionBlur(k=15, angle=[-5, 5]),
iaa.BlendAlphaCheckerboard(nb_rows=2, nb_cols=(1, 4),
foreground=iaa.AddToHue((-10, 10))),
iaa.BlendAlphaElementwise((0, 1.0), iaa.AddToHue(10)),
iaa.BilateralBlur(
d=(3, 10), sigma_color=(1, 5), sigma_space=(1, 5)),
iaa.AdditiveGaussianNoise(scale=0.02 * 255),
iaa.AddElementwise((-5, 5), per_channel=0.5),
iaa.AdditiveLaplaceNoise(scale=0.01 * 255),
iaa.AdditivePoissonNoise(20),
iaa.Cutout(fill_mode="gaussian", fill_per_channel=True),
iaa.CoarseDropout(0.02, size_percent=0.1),
iaa.SaltAndPepper(0.1, per_channel=True),
iaa.JpegCompression(compression=(70, 99)),
iaa.ImpulseNoise(0.02),
iaa.Dropout(p=(0, 0.04)),
iaa.Sharpen(alpha=0.1),
]) # oneof
])
def heavy_aug_batch(self, imgpath, n):
if isinstance(imgpath, str):
img = cv2.imread(imgpath)
else:
img = imgpath
images = np.array([img for _ in range(n)], dtype=np.uint8)
sometimes = lambda aug: iaa.Sometimes(0.3, aug)
seq = iaa.Sequential(
[
#
# Apply the following augmenters to most images.
#
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop some of the images by 0-10% of their height/width
sometimes(iaa.Crop(percent=(0, 0.1))),
# Apply affine transformations to some of the images
# - scale to 80-120% of image height/width (each axis independently)
# - translate by -20 to +20 relative to height/width (per axis)
# - rotate by -45 to +45 degrees
# - shear by -16 to +16 degrees
# - order: use nearest neighbour or bilinear interpolation (fast)
# - mode: use any available mode to fill newly created pixels
# see API or scikit-image for which modes are available
# - cval: if the mode is constant, then use a random brightness
# for the newly created pixels (e.g. sometimes black,
# sometimes white)
sometimes(
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-45, 45),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL)),
#
# Execute 0 to 5 of the following (less important) augmenters per
# image. Don't execute all of them, as that would often be way too
# strong.
#
iaa.SomeOf(
(0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
sometimes(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(alpha=(0, 0.7),
direction=(0.0, 1.0)),
])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
iaa.Invert(0.05,
per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.LinearContrast((0.5, 2.0), per_channel=0.5),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5),
sigma=0.25)),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True)
],
# do all of the above augmentations in random order
random_order=True)
return seq(images=images)
#Example on how to use this code:
#import matplotlib.pyplot as plt
# (import this code)
#datagen = DataAugmentor()
#resimg = datagen.perform_rotate("C:\\Users\\Jared Habermehl\\Documents\\2020-Image-Super-Resolution\\big_im_SR.png")
#resimg = datagen.perform_flip(resimg)
#plt.axis("off")
#plt.imshow(cv2.cvtColor(resimg, cv2.COLOR_BGR2RGB))
#plt.show()
"""
使用imgaug库,放在ToTensor前
:param img: IAA.Sequence
"""
self.seq = seq
def __call__(self, img):
img = np.asarray(img)
img = self.seq.augment_image(img)
img = Image.fromarray(img)
return img
if __name__ == '__main__':
seq = iaa.Sequential([
# iaa.Crop(px=(1, 16), keep_size=False),
# iaa.Fliplr(0.5),
# iaa.GaussianBlur(sigma=(0, 3.0)),
iaa.CoarseDropout((0.03, 0.15), size_percent=(0.1, 0.4)),
])
aug = ImgAug(seq)
rawimg = Image.open(
'/home/cmf/Pictures/Screenshot from 2019-10-15 17-50-59.png')
# rawimg = cv2.resize(rawimg, (22))
rawimg = rawimg.resize((224, 224))
for i in range(10):
img = aug(rawimg)
# img.show('123')
plt.imshow(np.asarray(img))
plt.show()
