iaa.SomeOf(
4,
[
# 以下一共10个，随机选7个进行处理，也可以将7改为其他数值，继续对数据集进行扩充
# 边缘检测，将检测到的赋值0或者255然后叠在原图上
# sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(
# alpha=(0, 0.7), direction=(0.0, 1.0)
# ),
# ])),
# 将RGB变成灰度图然后乘alpha加在原图上
# iaa.Grayscale(alpha=(0.0, 1.0)),
# 扭曲图像的局部区域
sometimes(iaa.PiecewiseAffine(scale=(0.001, 0.005))),
# 每个像素随机加减-10到10之间的数
iaa.Add((-10, 10), per_channel=0.5),
# 中值模糊
iaa.MedianBlur(
k=1, name=None, deterministic=False,
random_state=None),
#锐化
iaa.Sharpen(alpha=0,
lightness=1,
name=None,
deterministic=False,
random_state=None),
# 从最邻近像素中取均值来扰动。
iaa.AverageBlur(
k=1, name=None, deterministic=False,
random_state=None),
# 0-0.05的数值，分别乘以图片的宽和高为剪裁的像素个数，保持原尺寸
# iaa.Crop(percent=(0.01, 0.01)),
# iaa.Affine(
# # 对图片进行仿射变化，scale缩放x,y取值，translate_percent左右上下移动
# # rotate为旋转角度，shear为剪切取值范围0-360
# scale={"x": (0.99, 1), "y": (0.99, 1)},
# translate_percent={"x": (-0.01, 0.01), "y": (-0.01, 0.01)},
# rotate=(-1, 1),
# shear=(-1, 1)),
# 20%的图片像素值乘以0.8-1.2中间的数值,用以增加图片明亮度或改变颜色
iaa.Multiply((0.8, 1.2), per_channel=0.2),
# 随机去掉一些像素点, 即把这些像素点变成0。
iaa.Dropout(p=0,
per_channel=False,
name=None,
deterministic=False,
random_state=None),
# 浮雕效果
# iaa.Emboss(alpha=0, strength=2, name=None, deterministic=False, random_state=None),
# loc 噪声均值，scale噪声方差，50%的概率，对图片进行添加白噪声并应用于每个通道
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.1 * 255), per_channel=0.3)
],
),
def _load_augmentation_aug_all():
""" Load image augmentation model """
def sometimes(aug):
return iaa.Sometimes(0.5, aug)
return iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop images by -5% to 10% of their height/width
sometimes(
iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode='constant',
pad_cval=(0, 255))),
sometimes(
iaa.Affine(
# scale images to 80-120% of their size, individually per axis
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
# translate by -20 to +20 percent (per axis)
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
# use nearest neighbour or bilinear interpolation (fast)
order=[0, 1],
# if mode is constant, use a cval between 0 and 255
cval=(0, 255),
# use any of scikit-image's warping modes
# (see 2nd image from the top for examples)
mode='constant')),
# execute 0 to 5 of the following (less important) augmenters per
# image don't execute all of them, as that would often be way too
# strong
iaa.SomeOf(
(0, 5),
[
# convert images into their superpixel representation
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
iaa.OneOf([
# blur images with a sigma between 0 and 3.0
iaa.GaussianBlur((0, 3.0)),
# blur image using local means with kernel sizes
# between 2 and 7
iaa.AverageBlur(k=(2, 7)),
# blur image using local medians with kernel sizes
# between 2 and 7
iaa.MedianBlur(k=(3, 11)),
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
# add gaussian noise to images
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.OneOf([
# randomly remove up to 10% of the pixels
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
# invert color channels
iaa.Invert(0.05, per_channel=True),
# change brightness of images (by -10 to 10 of original value)
iaa.Add((-10, 10), per_channel=0.5),
# change hue and saturation
iaa.AddToHueAndSaturation((-20, 20)),
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0)))
]),
# improve or worsen the contrast
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),
iaa.Grayscale(alpha=(0.0, 1.0)),
# move pixels locally around (with random strengths)
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# sometimes move parts of the image around
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
],
random_order=True)
def _create_augment_pipeline():
# augmentors by https://github.com/aleju/imgaug
sometimes = lambda aug: iaa.Sometimes(0.2, aug)
# Define our sequence of augmentation steps that will be applied to every image
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
aug_pipe = iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
#iaa.Flipud(0.2), # vertically flip 20% of all images
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 2),
[
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
}, ),
iaa.Affine(translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
}, ),
iaa.Affine(rotate=(-15, 15), ),
iaa.Affine(shear=(-15, 15)),
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0
)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)),
# blur image using local means (kernel sizes between 2 and 7)
iaa.MedianBlur(k=(3, 11)),
# blur image using local medians (kernel sizes between 2 and 7)
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# add gaussian noise
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
iaa.Add((-10, 10),
per_channel=0.5), # change brightness of images
iaa.Multiply(
(0.5, 1.5),
per_channel=0.5), # change brightness of images
iaa.ContrastNormalization(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
],
random_order=True)
],
random_order=True)
return aug_pipe
def __init__(self,
list_file,
train,
transform,
device,
little_train=False,
with_file_path=False,
B=2,
C=21,
test_mode=False):
print('data init')
self.train = train
self.transform = transform
self.fnames = []
self.boxes = []
self.labels = []
self.resize = 448
self.B = B
self.C = C
self.device = device
self._test = test_mode
self.with_file_path = with_file_path
self.img_augsometimes = lambda aug: iaa.Sometimes(0.25, aug)
self.augmentation = iaa.Sequential(
[
# augment without change bboxes
self.img_augsometimes(
iaa.SomeOf(
(1, 3),
[
iaa.Dropout([0.05, 0.2
]), # drop 5% or 20% of all pixels
iaa.Sharpen((0.1, .8)), # sharpen the image
# iaa.GaussianBlur(sigma=(2., 3.5)),
iaa.OneOf([
iaa.GaussianBlur(sigma=(2., 3.5)),
iaa.AverageBlur(k=(2, 5)),
iaa.BilateralBlur(d=(7, 12),
sigma_color=(10, 250),
sigma_space=(10, 250)),
iaa.MedianBlur(k=(3, 7)),
]),
iaa.AddElementwise((-50, 50)),
iaa.AdditiveGaussianNoise(scale=(0, 0.1 * 255)),
iaa.JpegCompression(compression=(80, 95)),
iaa.Multiply((0.5, 1.5)),
iaa.MultiplyElementwise((0.5, 1.5)),
iaa.ReplaceElementwise(0.05, [0, 255]),
# iaa.WithColorspace(to_colorspace="HSV", from_colorspace="RGB",
# children=iaa.WithChannels(2, iaa.Add((-10, 50)))),
iaa.OneOf([
iaa.WithColorspace(to_colorspace="HSV",
from_colorspace="RGB",
children=iaa.WithChannels(
1, iaa.Add((-10, 50)))),
iaa.WithColorspace(to_colorspace="HSV",
from_colorspace="RGB",
children=iaa.WithChannels(
2, iaa.Add((-10, 50)))),
]),
],
random_order=True)),
],
random_order=True)
# torch.manual_seed(23)
with open(list_file) as f:
lines = f.readlines()
if little_train:
lines = lines[:64 * 8]
for line in lines:
splited = line.strip().split()
self.fnames.append(splited[0])
self.num_samples = len(self.fnames)
def __init__(self, images,
config,
shuffle=True,
jitter=True,
norm=None):
self.generator = None
self.images = images
self.config = config
self.shuffle = shuffle
self.jitter = jitter
self.norm = norm
self.anchors = [BoundBox(0, 0, config['ANCHORS'][2*i], config['ANCHORS'][2*i+1]) for i in range(int(len(config['ANCHORS'])//2))]
### augmentors by https://github.com/aleju/imgaug
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
self.aug_pipe = iaa.Sequential(
[
# apply the following augmenters to most images
#iaa.Fliplr(0.5), # horizontally flip 50% of all images
#iaa.Flipud(0.2), # vertically flip 20% of all images
#sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
sometimes(iaa.Affine(
#scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
#translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
#rotate=(-5, 5), # rotate by -45 to +45 degrees
#shear=(-5, 5), # shear by -16 to +16 degrees
#order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
#cval=(0, 255), # if mode is constant, use a cval between 0 and 255
#mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
#sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
#iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges
#sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
#])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
#iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
#iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
iaa.Multiply((0.5, 1.5), per_channel=0.5), # change brightness of images (50-150% of original value)
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
#iaa.Grayscale(alpha=(0.0, 1.0)),
#sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
#sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
],
random_order=True
)
],
random_order=True
)
if shuffle: np.random.shuffle(self.images)
def test_dtype_preservation():
reseed()
size = (4, 16, 16, 3)
images = [
np.random.uniform(0, 255, size).astype(np.uint8),
np.random.uniform(0, 65535, size).astype(np.uint16),
np.random.uniform(0, 4294967295, size).astype(np.uint32),
np.random.uniform(-128, 127, size).astype(np.int16),
np.random.uniform(-32768, 32767, size).astype(np.int32),
np.random.uniform(0.0, 1.0, size).astype(np.float32),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float16),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float32),
np.random.uniform(-1000.0, 1000.0, size).astype(np.float64)
]
default_dtypes = set([arr.dtype for arr in images])
# Some dtypes are here removed per augmenter, because the respective
# augmenter does not support them. This test currently only checks whether
# dtypes are preserved from in- to output for all dtypes that are supported
# per augmenter.
# dtypes are here removed via list comprehension instead of
# `default_dtypes - set([dtype])`, because the latter one simply never
# removed the dtype(s) for some reason?!
def _not_dts(dts):
return [dt for dt in default_dtypes if dt not in dts]
augs = [
(iaa.Add((-5, 5),
name="Add"), _not_dts([np.uint32, np.int32, np.float64])),
(iaa.AddElementwise((-5, 5), name="AddElementwise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.AdditiveGaussianNoise(0.01 * 255, name="AdditiveGaussianNoise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Multiply((0.95, 1.05), name="Multiply"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Dropout(0.01, name="Dropout"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.CoarseDropout(0.01, size_px=6, name="CoarseDropout"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Invert(0.01, per_channel=True, name="Invert"), default_dtypes),
(iaa.ContrastNormalization(
(0.95, 1.05), name="ContrastNormalization"), default_dtypes),
(iaa.GaussianBlur(sigma=(0.95, 1.05),
name="GaussianBlur"), _not_dts([np.float16])),
(iaa.AverageBlur((3, 5), name="AverageBlur"),
_not_dts([np.uint32, np.int32, np.float16])),
(iaa.MedianBlur((3, 5), name="MedianBlur"),
_not_dts([np.uint32, np.int32, np.float16, np.float64])),
(iaa.BilateralBlur((3, 5), name="BilateralBlur"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float16, np.float64
])),
# WithColorspace ?
# iaa.AddToHueAndSaturation((-5, 5), name="AddToHueAndSaturation"), # works only with RGB/uint8
# ChangeColorspace ?
# iaa.Grayscale((0.0, 0.1), name="Grayscale"), # works only with RGB/uint8
# Convolve ?
(iaa.Sharpen((0.0, 0.1), lightness=(1.0, 1.2), name="Sharpen"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.Emboss(alpha=(0.0, 0.1), strength=(0.5, 1.5), name="Emboss"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.EdgeDetect(alpha=(0.0, 0.1), name="EdgeDetect"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.DirectedEdgeDetect(alpha=(0.0, 0.1),
direction=0,
name="DirectedEdgeDetect"),
_not_dts([np.uint32, np.int32, np.float16, np.uint32])),
(iaa.Fliplr(0.5, name="Fliplr"), default_dtypes),
(iaa.Flipud(0.5, name="Flipud"), default_dtypes),
(iaa.Affine(translate_px=(-5, 5), name="Affine-translate-px"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(translate_percent=(-0.05, 0.05),
name="Affine-translate-percent"),
_not_dts([np.uint32, np.int32])),
(iaa.Affine(rotate=(-20, 20),
name="Affine-rotate"), _not_dts([np.uint32, np.int32])),
(iaa.Affine(shear=(-20, 20),
name="Affine-shear"), _not_dts([np.uint32, np.int32])),
(iaa.Affine(scale=(0.9, 1.1),
name="Affine-scale"), _not_dts([np.uint32, np.int32])),
(iaa.PiecewiseAffine(scale=(0.001, 0.005),
name="PiecewiseAffine"), default_dtypes),
# (iaa.PerspectiveTransform(scale=(0.01, 0.10), name="PerspectiveTransform"), not_dts([np.uint32])),
(iaa.ElasticTransformation(alpha=(0.1, 0.2),
sigma=(0.1, 0.2),
name="ElasticTransformation"),
_not_dts([np.float16])),
(iaa.Sequential([iaa.Noop(), iaa.Noop()],
name="SequentialNoop"), default_dtypes),
(iaa.SomeOf(1, [iaa.Noop(), iaa.Noop()],
name="SomeOfNoop"), default_dtypes),
(iaa.OneOf([iaa.Noop(), iaa.Noop()],
name="OneOfNoop"), default_dtypes),
(iaa.Sometimes(0.5, iaa.Noop(), name="SometimesNoop"), default_dtypes),
(iaa.Sequential([iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))],
name="Sequential"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.SomeOf(1, [iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))],
name="SomeOf"), _not_dts([np.uint32, np.int32,
np.float64])),
(iaa.OneOf([iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))],
name="OneOf"), _not_dts([np.uint32, np.int32, np.float64])),
(iaa.Sometimes(0.5, iaa.Add((-5, 5)), name="Sometimes"),
_not_dts([np.uint32, np.int32, np.float64])),
# WithChannels
(iaa.Noop(name="Noop"), default_dtypes),
# Lambda
# AssertLambda
# AssertShape
(iaa.Alpha((0.0, 0.1), iaa.Noop(), name="AlphaNoop"), default_dtypes),
(iaa.AlphaElementwise((0.0, 0.1),
iaa.Noop(),
name="AlphaElementwiseNoop"), default_dtypes),
(iaa.SimplexNoiseAlpha(iaa.Noop(),
name="SimplexNoiseAlphaNoop"), default_dtypes),
(iaa.FrequencyNoiseAlpha(exponent=(-2, 2),
first=iaa.Noop(),
name="SimplexNoiseAlphaNoop"),
default_dtypes),
(iaa.Alpha((0.0, 0.1), iaa.Add(10),
name="Alpha"), _not_dts([np.uint32, np.int32, np.float64])),
(iaa.AlphaElementwise((0.0, 0.1), iaa.Add(10),
name="AlphaElementwise"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.SimplexNoiseAlpha(iaa.Add(10), name="SimplexNoiseAlpha"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.FrequencyNoiseAlpha(exponent=(-2, 2),
first=iaa.Add(10),
name="SimplexNoiseAlpha"),
_not_dts([np.uint32, np.int32, np.float64])),
(iaa.Superpixels(p_replace=0.01, n_segments=64),
_not_dts([np.float16, np.float32, np.float64])),
(iaa.Resize({
"height": 4,
"width": 4
}, name="Resize"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
])),
(iaa.CropAndPad(px=(-10, 10), name="CropAndPad"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
])),
(iaa.Pad(px=(0, 10), name="Pad"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
])),
(iaa.Crop(px=(0, 10), name="Crop"),
_not_dts([
np.uint16, np.uint32, np.int16, np.int32, np.float32, np.float16,
np.float64
]))
]
for (aug, allowed_dtypes) in augs:
# print("aug", aug.name)
# print("allowed_dtypes", allowed_dtypes)
for images_i in images:
if images_i.dtype in allowed_dtypes:
# print("image dt", images_i.dtype)
images_aug = aug.augment_images(images_i)
assert images_aug.dtype == images_i.dtype
else:
# print("image dt", images_i.dtype, "[SKIPPED]")
pass
y[i, ] = mask_oh
return X, y
# Training Data Configuration
# Data Path
image_path = '../sliced_img/'
mask_path = '../sliced_mask/'
train_data_dir = '../train_data.csv'
valid_data_dir = '../valid_data.csv'
augmentation = iaa.SomeOf((0, 3), [
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Noop(),
iaa.OneOf([
iaa.Affine(rotate=90),
iaa.Affine(rotate=180),
iaa.Affine(rotate=270)
]),
iaa.GaussianBlur(sigma=(0.0, 0.5)),
])
# Parameters
train_transform_params = {
'image_size': (512, 512),
'target_image_size': (512, 512),
'batch_size': 32,
'n_classes': 3,
'n_channels': 3,
'shuffle': True,
'transform': augmentation
}
def main():
args = parser.parse_args()
# Data augmentation
global seq_geo
global seq_img
seq_geo = iaa.SomeOf(
(0, 5),
[
iaa.Fliplr(0.5), # horizontally flip 50% of the images
iaa.PerspectiveTransform(scale=(0, 0.075)),
iaa.Affine(
scale={
"x": (0.8, 1.0),
"y": (0.8, 1.0)
},
rotate=(-5, 5),
translate_percent={
"x": (-0.1, 0.1),
"y": (-0.1, 0.1)
},
), # rotate by -45 to +45 degrees),
iaa.Crop(percent=(
0, 0.125
)), # crop images from each side by 0 to 12.5% (randomly chosen)
iaa.CoarsePepper(p=0.01, size_percent=0.1)
],
random_order=False)
# Content transformation
seq_img = iaa.SomeOf(
(0, 3),
[
iaa.GaussianBlur(
sigma=(0, 1.0)), # blur images with a sigma of 0 to 2.0
iaa.ContrastNormalization(alpha=(0.9, 1.1)),
iaa.Grayscale(alpha=(0, 0.2)),
iaa.Multiply((0.9, 1.1))
])
# We store all arguments in a json file. This has two advantages:
# 1. We can always get back and see what exactly that experiment was
# 2. We can resume an experiment as-is without needing to remember all flags.
args_file = os.path.join(args.experiment_root, 'args.json')
if args.resume:
if not os.path.isfile(args_file):
raise IOError('`args.json` not found in {}'.format(args_file))
print('Loading args from {}.'.format(args_file))
with open(args_file, 'r') as f:
args_resumed = json.load(f)
args_resumed['resume'] = True # This would be overwritten.
# When resuming, we not only want to populate the args object with the
# values from the file, but we also want to check for some possible
# conflicts between loaded and given arguments.
for key, value in args.__dict__.items():
if key in args_resumed:
resumed_value = args_resumed[key]
if resumed_value != value:
print('Warning: For the argument `{}` we are using the'
' loaded value `{}`. The provided value was `{}`'
'.'.format(key, resumed_value, value))
args.__dict__[key] = resumed_value
else:
print('Warning: A new argument was added since the last run:'
' `{}`. Using the new value: `{}`.'.format(key, value))
else:
# If the experiment directory exists already, we bail in fear.
if os.path.exists(args.experiment_root):
if os.listdir(args.experiment_root):
print('The directory {} already exists and is not empty.'
' If you want to resume training, append --resume to'
' your call.'.format(args.experiment_root))
exit(1)
else:
os.makedirs(args.experiment_root)
# Store the passed arguments for later resuming and grepping in a nice
# and readable format.
with open(args_file, 'w') as f:
json.dump(vars(args),
f,
ensure_ascii=False,
indent=2,
sort_keys=True)
log_file = os.path.join(args.experiment_root, "train")
logging.config.dictConfig(common.get_logging_dict(log_file))
log = logging.getLogger('train')
# Also show all parameter values at the start, for ease of reading logs.
log.info('Training using the following parameters:')
for key, value in sorted(vars(args).items()):
log.info('{}: {}'.format(key, value))
# Check them here, so they are not required when --resume-ing.
if not args.train_set:
parser.print_help()
log.error("You did not specify the `train_set` argument!")
sys.exit(1)
if not args.image_root:
parser.print_help()
log.error("You did not specify the required `image_root` argument!")
sys.exit(1)
# Load the data from the CSV file.
pids, fids = common.load_dataset(args.train_set, args.image_root)
max_fid_len = max(map(len, fids)) # We'll need this later for logfiles.
# Load feature embeddings
if args.hard_pool_size > 0:
with h5py.File(args.train_embeddings, 'r') as f_train:
train_embs = np.array(f_train['emb'])
f_dists = scipy.spatial.distance.cdist(train_embs, train_embs)
hard_ids = get_hard_id_pool(pids, f_dists, args.hard_pool_size)
# Setup a tf.Dataset where one "epoch" loops over all PIDS.
# PIDS are shuffled after every epoch and continue indefinitely.
unique_pids = np.unique(pids)
dataset = tf.data.Dataset.from_tensor_slices(unique_pids)
dataset = dataset.shuffle(len(unique_pids))
# Constrain the dataset size to a multiple of the batch-size, so that
# we don't get overlap at the end of each epoch.
if args.hard_pool_size == 0:
dataset = dataset.take(
(len(unique_pids) // args.batch_p) * args.batch_p)
dataset = dataset.repeat(
None) # Repeat forever. Funny way of stating it.
else:
dataset = dataset.repeat(
None) # Repeat forever. Funny way of stating it.
dataset = dataset.map(lambda pid: sample_batch_ids_for_pid(
pid, all_pids=pids, batch_p=args.batch_p, all_hard_pids=hard_ids))
# Unbatch the P PIDs
dataset = dataset.apply(tf.contrib.data.unbatch())
# For every PID, get K images.
dataset = dataset.map(lambda pid: sample_k_fids_for_pid(
pid, all_fids=fids, all_pids=pids, batch_k=args.batch_k))
# Ungroup/flatten the batches for easy loading of the files.
dataset = dataset.apply(tf.contrib.data.unbatch())
# Convert filenames to actual image tensors.
net_input_size = (args.net_input_height, args.net_input_width)
pre_crop_size = (args.pre_crop_height, args.pre_crop_width)
dataset = dataset.map(lambda fid, pid: common.fid_to_image(
fid,
pid,
image_root=args.image_root,
image_size=pre_crop_size if args.crop_augment else net_input_size),
num_parallel_calls=args.loading_threads)
# Augment the data if specified by the arguments.
if args.augment == False:
dataset = dataset.map(
lambda im, fid, pid: common.fid_to_image(
fid,
pid,
image_root=args.image_root,
image_size=pre_crop_size
if args.crop_augment else net_input_size), # Ergys
num_parallel_calls=args.loading_threads)
if args.flip_augment:
dataset = dataset.map(lambda im, fid, pid: (
tf.image.random_flip_left_right(im), fid, pid))
if args.crop_augment:
dataset = dataset.map(lambda im, fid, pid: (tf.random_crop(
im, net_input_size + (3, )), fid, pid))
else:
dataset = dataset.map(lambda im, fid, pid: common.fid_to_image(
fid, pid, image_root=args.image_root, image_size=net_input_size),
num_parallel_calls=args.loading_threads)
dataset = dataset.map(lambda im, fid, pid: (tf.py_func(
augment_images, [im], [tf.float32]), fid, pid))
dataset = dataset.map(lambda im, fid, pid: (tf.reshape(
im[0],
(args.net_input_height, args.net_input_width, 3)), fid, pid))
# Group it back into PK batches.
batch_size = args.batch_p * args.batch_k
dataset = dataset.batch(batch_size)
# Overlap producing and consuming for parallelism.
dataset = dataset.prefetch(batch_size * 2)
# Since we repeat the data infinitely, we only need a one-shot iterator.
images, fids, pids = dataset.make_one_shot_iterator().get_next()
print(images)
# Create the model and an embedding head.
model = import_module('nets.' + args.model_name)
head = import_module('heads.' + args.head_name)
# Feed the image through the model. The returned `body_prefix` will be used
# further down to load the pre-trained weights for all variables with this
# prefix.
endpoints, body_prefix = model.endpoints(images, is_training=True)
with tf.name_scope('head'):
endpoints = head.head(endpoints, args.embedding_dim, is_training=True)
# Create the loss in two steps:
# 1. Compute all pairwise distances according to the specified metric.
# 2. For each anchor along the first dimension, compute its loss.
dists = loss.cdist(endpoints['emb'], endpoints['emb'], metric=args.metric)
losses, train_top1, prec_at_k, _, neg_dists, pos_dists = loss.LOSS_CHOICES[
args.loss](dists,
pids,
args.margin,
batch_precision_at_k=args.batch_k - 1)
# Count the number of active entries, and compute the total batch loss.
num_active = tf.reduce_sum(tf.cast(tf.greater(losses, 1e-5), tf.float32))
loss_mean = tf.reduce_mean(losses)
# Some logging for tensorboard.
tf.summary.histogram('loss_distribution', losses)
tf.summary.scalar('loss', loss_mean)
tf.summary.scalar('batch_top1', train_top1)
tf.summary.scalar('batch_prec_at_{}'.format(args.batch_k - 1), prec_at_k)
tf.summary.scalar('active_count', num_active)
tf.summary.histogram('embedding_dists', dists)
tf.summary.histogram('embedding_pos_dists', pos_dists)
tf.summary.histogram('embedding_neg_dists', neg_dists)
tf.summary.histogram('embedding_lengths',
tf.norm(endpoints['emb_raw'], axis=1))
# Create the mem-mapped arrays in which we'll log all training detail in
# addition to tensorboard, because tensorboard is annoying for detailed
# inspection and actually discards data in histogram summaries.
if args.detailed_logs:
log_embs = lb.create_or_resize_dat(
os.path.join(args.experiment_root, 'embeddings'),
dtype=np.float32,
shape=(args.train_iterations, batch_size, args.embedding_dim))
log_loss = lb.create_or_resize_dat(
os.path.join(args.experiment_root, 'losses'),
dtype=np.float32,
shape=(args.train_iterations, batch_size))
log_fids = lb.create_or_resize_dat(
os.path.join(args.experiment_root, 'fids'),
dtype='S' + str(max_fid_len),
shape=(args.train_iterations, batch_size))
# These are collected here before we add the optimizer, because depending
# on the optimizer, it might add extra slots, which are also global
# variables, with the exact same prefix.
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
body_prefix)
# Define the optimizer and the learning-rate schedule.
# Unfortunately, we get NaNs if we don't handle no-decay separately.
# modified by ha (default decay rate 0.001)
global_step = tf.Variable(0, name='global_step', trainable=False)
if 0 <= args.decay_start_iteration < args.train_iterations:
learning_rate = tf.train.exponential_decay(
args.learning_rate,
tf.maximum(0, global_step - args.decay_start_iteration),
args.train_iterations - args.decay_start_iteration, 0.001)
else:
learning_rate = args.learning_rate
tf.summary.scalar('learning_rate', learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
# Feel free to try others!
# optimizer = tf.train.AdadeltaOptimizer(learning_rate)
# Update_ops are used to update batchnorm stats.
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
train_op = optimizer.minimize(loss_mean, global_step=global_step)
# Define a saver for the complete model.
checkpoint_saver = tf.train.Saver(max_to_keep=0)
with tf.Session() as sess:
if args.resume:
# In case we're resuming, simply load the full checkpoint to init.
last_checkpoint = tf.train.latest_checkpoint(args.experiment_root)
log.info('Restoring from checkpoint: {}'.format(last_checkpoint))
checkpoint_saver.restore(sess, last_checkpoint)
else:
# But if we're starting from scratch, we may need to load some
# variables from the pre-trained weights, and random init others.
sess.run(tf.global_variables_initializer())
if args.initial_checkpoint is not None:
# saver = tf.train.Saver(model_variables)
saver = tf.train.import_meta_graph(
'./mobilenet/mobilenet_v2_1.4_224.ckpt.meta')
saver.restore(sess, args.initial_checkpoint)
# In any case, we also store this initialization as a checkpoint,
# such that we could run exactly reproduceable experiments.
checkpoint_saver.save(sess,
os.path.join(args.experiment_root,
'checkpoint'),
global_step=0)
merged_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(args.experiment_root,
sess.graph)
start_step = sess.run(global_step)
log.info('Starting training from iteration {}.'.format(start_step))
# Finally, here comes the main-loop. This `Uninterrupt` is a handy
# utility such that an iteration still finishes on Ctrl+C and we can
# stop the training cleanly.
with lb.Uninterrupt(sigs=[SIGINT, SIGTERM], verbose=True) as u:
for i in range(start_step, args.train_iterations):
# Compute gradients, update weights, store logs!
start_time = time.time()
_, summary, step, b_prec_at_k, b_embs, b_loss, b_fids = \
sess.run([train_op, merged_summary, global_step,
prec_at_k, endpoints['emb'], losses, fids])
elapsed_time = time.time() - start_time
# Compute the iteration speed and add it to the summary.
# We did observe some weird spikes that we couldn't track down.
summary2 = tf.Summary()
summary2.value.add(tag='secs_per_iter',
simple_value=elapsed_time)
summary_writer.add_summary(summary2, step)
summary_writer.add_summary(summary, step)
if args.detailed_logs:
log_embs[i], log_loss[i], log_fids[
i] = b_embs, b_loss, b_fids
# Do a huge print out of the current progress.
seconds_todo = (args.train_iterations - step) * elapsed_time
log.info(
'iter:{:6d}, loss min|avg|max: {:.3f}|{:.3f}|{:6.3f}, '
'batch-p@{}: {:.2%}, ETA: {} ({:.2f}s/it)'.format(
step, float(np.min(b_loss)), float(np.mean(b_loss)),
float(np.max(b_loss)), args.batch_k - 1,
float(b_prec_at_k),
timedelta(seconds=int(seconds_todo)), elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
# Save a checkpoint of training every so often.
if (args.checkpoint_frequency > 0
and step % args.checkpoint_frequency == 0):
checkpoint_saver.save(sess,
os.path.join(args.experiment_root,
'checkpoint'),
global_step=step)
# Stop the main-loop at the end of the step, if requested.
if u.interrupted:
log.info("Interrupted on request!")
break
# Store one final checkpoint. This might be redundant, but it is crucial
# in case intermediate storing was disabled and it saves a checkpoint
# when the process was interrupted.
checkpoint_saver.save(sess,
os.path.join(args.experiment_root, 'checkpoint'),
global_step=step)
}, #平移±20%之间
rotate=(-25, 25), #旋转±45度之间
shear=(-2, 2), #剪切变换±16度，（矩形变平行四边形）
order=[0, 1], #使用最邻近差值或者双线性差值
cval=(0, 255), #全白全黑填充
mode=imgaug.ALL #定义填充图像外区域的方法
)),
iaa.SomeOf(
(0, 5), # 使用下面的0个到5个之间的方法增强图像
[
sometimes( # 将部分图像进行超像素的表示
iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))),
iaa.OneOf([ #用高斯模糊，均值模糊，中值模糊中的一种增强
iaa.GaussianBlur((0, 2.0)),
iaa.AverageBlur(k=(2, 7)), # 核大小2~7之间
iaa.MedianBlur(k=(3, 7)),
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), #锐化
iaa.AdditiveGaussianNoise( # 高斯噪声
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # 扭曲图像的局部
],
random_order=True # 随机顺序
)
],
random_order=True # 随机顺序
)
imglist = []
WSI_MASK_PATH = './1//' #存放图片的文件夹路径
paths = glob.glob(os.path.join(WSI_MASK_PATH, '*.jpg'))
paths.sort()
iaa.SomeOf(
(3, 4),
[
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.Add((-10, 10)),
iaa.Sometimes(
0.5,
iaa.SomeOf(
1, [iaa.Scale(0.5),
iaa.Scale(0.8),
iaa.Scale(2.0)])),
iaa.Sometimes(
0.5,
iaa.SomeOf(1, [
iaa.Lambda(func_images=rotate_45,
func_keypoints=func_keypoints),
iaa.Lambda(func_images=rotate_135,
func_keypoints=func_keypoints),
iaa.Lambda(func_images=rotate_225,
func_keypoints=func_keypoints),
iaa.Lambda(func_images=rotate_315,
func_keypoints=func_keypoints),
])),
iaa.Sharpen(),
iaa.SomeOf(1,
[iaa.Fliplr(1.0), iaa.Flipud(1.0)]),
iaa.OneOf([
iaa.AverageBlur(k=(1, 3)),
iaa.MedianBlur(k=(1, 3)),
iaa.GaussianBlur(sigma=iap.Uniform(0.8, 1.2)),
]),
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5) # 改变对比度
])
def augment(gen, config, keep=False, stages=None, params=None, meta_udf=None):
'''Augments the dataset if required.
This function pays special respect to objects of type `box-array` to update them according to the transformations.
It will also update the `complexity` (global value) accordingly.
The `meta_udf` has a signature of `(mdata, gdata, config, params) -> (mdata, gdata)` in order to allow the user to define own transformations
on the metadata.
Args:
gen (Generator): Beard-Style generator that should be augmented
config (dict): Configuration of the dataset
stages (list): List of dataset-stages (dev, train, etc.) that should be augmented (None=Augment all)
keep (bool): Defines if the original image should be preserved
params (dict): Dict of all relevant elements
meta_udf (fct): user defined function that allows to update use-case specific metadata. Signature: (mdata, gdata, transform) => (mdata)
'''
# import relevant libs (do here, to avoid global crash if not installed!)
import imgaug as ia
from imgaug import augmenters as iaa
# generate list of all augmentations
augs = []
if params is not None:
if "flip" in params:
augs.append(iaa.Fliplr(params["flip"]))
if 'blur' in params:
augs.append( iaa.GaussianBlur(sigma=(0, params["blur"])) )
if 'crop' in params:
augs.append( iaa.Crop(px=(0, params['crop'])) )
if 'contrast' in params:
augs.append( iaa.ContrastNormalization((params['contrast'][0], params['contrast'][1])) )
if 'noise' in params:
# strenght of the noise and amount added
augs.append( iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, params['noise'][0]*255), per_channel=params['noise'][1]) )
if 'transform' in params:
transform_prob = lambda aug: iaa.Sometimes(params['transform'], aug)
shear = params['shear'] if 'shear' in params else 0
rot = params['rotate'] if 'rotate' in params else 0
trans = params['translate'] if 'translate' in params else 0
scale = params['scale'] if 'scale' in params else 0
augs.append( transform_prob(iaa.Affine(
scale={"x": (scale[0], scale[1]), "y": (scale[0], scale[1])},
translate_percent={"x": (-trans, trans), "y": (-trans, trans)},
rotate=(-rot, rot),
shear=(-shear, shear),
order=[0, 1],
cval=(0, 255),
mode=['constant', 'edge']
)) )
# create the augmentation model
seq = iaa.Sequential(iaa.SomeOf((min(1, len(augs)), None), augs), random_order=True)
# iterate through all data
for img, gdata, mdata, btype in gen:
# check if stage is augmented
if stages is not None and btype not in stages:
yield img, gdata, mdata, btype
continue
# output original if keep
if keep:
yield img, gdata, mdata, btype
orig = img
# iterate through all images that shall be generated
for _ in range(params["per_img"]):
# generate copy to not add multiple augmentations
img = np.copy(orig)
# retrieve augmentation standard
seq_det = seq.to_deterministic()
# augment the image
aug_img = seq_det.augment_images([img])[0]
# TODO: update metadata (todo: update for better search)
aug_mdata = [x.copy() for x in mdata]
bbs_dict = {const.ITEM_BBOX: [x[const.ITEM_BBOX] for x in aug_mdata]}
# iterate through all elements
for key in bbs_dict:
bbs = np.copy(np.array(bbs_dict[key]))
if len(bbs.shape) < 2 or bbs.shape[1] < 4:
print("\nERROR: WRONG BBS")
print(bbs)
continue
# TODO: implement filter?
#fltr = np.logical_and(bbs[:, 2] > bbs[:, 0], bbs[:, 3] > bbs[:, 1])
#bbs = bbs[fltr]
#cls = np.array(meta[0])[fltr]
# TODO: implement order and format?
# convert the bounding boxes to format
bbs = [ia.BoundingBox(x1=bb[0], y1=bb[1], x2=bb[2], y2=bb[3]) for bb in bbs if bb[2] > bb[0] and bb[3] > bb[1]]
bbs = ia.BoundingBoxesOnImage(bbs, shape=img.shape)
# augment the bounding boxes
aug_bbs = seq_det.augment_bounding_boxes([bbs])[0]
aug_bbs = np.array([[bb.x1, bb.y1, bb.x2, bb.y2] for bb in aug_bbs.bounding_boxes])
# update bounding boxes
# load bounding boxes / check format
for i in range(len(aug_bbs)):
aug_mdata[i][key] = aug_bbs[i]
# TODO: update landmarks
# TODO: update complexity based on transformations?
# perform UDF transformations
if meta_udf is not None:
# TODO: update
mdata = meta_udf(mdata, gdata, config, params)
# send to gen
yield aug_img, gdata, aug_mdata, btype
def augmentation(image, mask):
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential(
[
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop images by -5% to 10% of their height/width
sometimes(
iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode=ia.ALL,
pad_cval=(0, 255))),
sometimes(
iaa.Affine(
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
# scale images to 80-120% of their size, individually per axis
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
# translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[
0,
1
], # use nearest neighbour or bilinear interpolation (fast)
cval=(
0, 255
), # if mode is constant, use a cval between 0 and 255
mode=ia.
ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
iaa.SomeOf(
(0, 5),
[
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
# convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0)
), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)),
# blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)),
# blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 2.0)), # emboss images
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
iaa.Invert(0.05,
per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5),
iaa.AddToHueAndSaturation(
(-20, 20)), # change hue and saturation
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
]),
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5),
sigma=0.25)),
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
],
random_order=True)
image_heavy, mask_heavy = seq(images=image, segmentation_maps=mask)
return image_heavy, mask_heavy
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = BalloonDataset()
dataset_train.load_balloon(args.dataset, "train")
dataset_train.prepare()
# Validation dataset
dataset_val = BalloonDataset()
dataset_val.load_balloon(args.dataset, "val")
dataset_val.prepare()
# Image augmentation
# http://imgaug.readthedocs.io/en/latest/source/augmenters.html
augmentation = iaa.SomeOf(
(1, 5),
[
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Affine(rotate=90), # new to force more horizontal lines
iaa.Affine(
rotate=(-90, 90), mode="edge"
), # mode= "edge" ads straight lines in created empty space
#for this do one of
iaa.OneOf([
iaa.CropAndPad(percent=(-0.3, 0.05),
sample_independently=False,
pad_mode="edge"),
iaa.Crop(percent=(0, 0.05))
]),
iaa.GaussianBlur(sigma=(0.0, 1.0)),
iaa.Multiply((0.5, 1.2)), #changeing brightness
iaa.Grayscale(alpha=(0.0, 0.9))
])
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
print("Training network heads")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=15,
augmentation=augmentation,
layers='heads') # 'heads' or 'all'
print("Training 4+")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=60,
augmentation=augmentation,
layers='4+') # 'heads' or 'all'
print("Train all")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=200,
augmentation=augmentation,
layers='all') # 'heads' or 'all'
print("Train all lower learning rate")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE / 10,
epochs=400,
augmentation=augmentation,
layers='all') # 'heads' or 'all'
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
iaa.SomeOf((3, 4), [
iaa.Add((-40, 40), per_channel=0.5),
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.Multiply((0.5, 1)),
iaa.JpegCompression(compression=(70, 99)),
iaa.BlendAlphaHorizontalLinearGradient(iaa.AddToHue((-100, 100))),
iaa.BlendAlphaCheckerboard(nb_rows=2,
nb_cols=(1, 4),
foreground=iaa.AddToHue((-100, 100))),
iaa.GaussianBlur(sigma=(0.0, 1.0)),
iaa.AddToHueAndSaturation((-50, 50), per_channel=True),
iaa.GammaContrast((0.5, 2.0), per_channel=True),
iaa.Affine(rotate=(-15, 15)),
iaa.Affine(
translate_percent={"x": -0.20}, mode=ia.ALL, cval=(0, 255)),
iaa.Affine(
translate_percent={"y": -0.20}, mode=ia.ALL, cval=(0, 255)),
iaa.Affine(
translate_percent={"x": +0.20}, mode=ia.ALL, cval=(0, 255)),
iaa.Affine(
translate_percent={"y": +0.20}, mode=ia.ALL, cval=(0, 255)),
],
random_order=True)
],
random_order=True)
images_aug = seq(images=images)
def test_unusual_channel_numbers():
reseed()
images = [(0, create_random_images((4, 16, 16))),
(1, create_random_images((4, 16, 16, 1))),
(2, create_random_images((4, 16, 16, 2))),
(4, create_random_images((4, 16, 16, 4))),
(5, create_random_images((4, 16, 16, 5))),
(10, create_random_images((4, 16, 16, 10))),
(20, create_random_images((4, 16, 16, 20)))]
augs = [
iaa.Add((-5, 5), name="Add"),
iaa.AddElementwise((-5, 5), name="AddElementwise"),
iaa.AdditiveGaussianNoise(0.01 * 255, name="AdditiveGaussianNoise"),
iaa.Multiply((0.95, 1.05), name="Multiply"),
iaa.Dropout(0.01, name="Dropout"),
iaa.CoarseDropout(0.01, size_px=6, name="CoarseDropout"),
iaa.Invert(0.01, per_channel=True, name="Invert"),
iaa.ContrastNormalization((0.95, 1.05), name="ContrastNormalization"),
iaa.GaussianBlur(sigma=(0.95, 1.05), name="GaussianBlur"),
iaa.AverageBlur((3, 5), name="AverageBlur"),
iaa.MedianBlur((3, 5), name="MedianBlur"),
# iaa.BilateralBlur((3, 5), name="BilateralBlur"), # works only with 3/RGB channels
# WithColorspace ?
# iaa.AddToHueAndSaturation((-5, 5), name="AddToHueAndSaturation"), # works only with 3/RGB channels
# ChangeColorspace ?
# iaa.Grayscale((0.0, 0.1), name="Grayscale"), # works only with 3 channels
# Convolve ?
iaa.Sharpen((0.0, 0.1), lightness=(1.0, 1.2), name="Sharpen"),
iaa.Emboss(alpha=(0.0, 0.1), strength=(0.5, 1.5), name="Emboss"),
iaa.EdgeDetect(alpha=(0.0, 0.1), name="EdgeDetect"),
iaa.DirectedEdgeDetect(alpha=(0.0, 0.1),
direction=0,
name="DirectedEdgeDetect"),
iaa.Fliplr(0.5, name="Fliplr"),
iaa.Flipud(0.5, name="Flipud"),
iaa.Affine(translate_px=(-5, 5), name="Affine-translate-px"),
iaa.Affine(translate_percent=(-0.05, 0.05),
name="Affine-translate-percent"),
iaa.Affine(rotate=(-20, 20), name="Affine-rotate"),
iaa.Affine(shear=(-20, 20), name="Affine-shear"),
iaa.Affine(scale=(0.9, 1.1), name="Affine-scale"),
iaa.PiecewiseAffine(scale=(0.001, 0.005), name="PiecewiseAffine"),
iaa.PerspectiveTransform(scale=(0.01, 0.10),
name="PerspectiveTransform"),
iaa.ElasticTransformation(alpha=(0.1, 0.2),
sigma=(0.1, 0.2),
name="ElasticTransformation"),
iaa.Sequential([iaa.Add((-5, 5)),
iaa.AddElementwise((-5, 5))]),
iaa.SomeOf(1, [iaa.Add(
(-5, 5)), iaa.AddElementwise((-5, 5))]),
iaa.OneOf([iaa.Add((-5, 5)),
iaa.AddElementwise((-5, 5))]),
iaa.Sometimes(0.5, iaa.Add((-5, 5)), name="Sometimes"),
# WithChannels
iaa.Noop(name="Noop"),
# Lambda
# AssertLambda
# AssertShape
iaa.Alpha((0.0, 0.1), iaa.Add(10), name="Alpha"),
iaa.AlphaElementwise((0.0, 0.1), iaa.Add(10), name="AlphaElementwise"),
iaa.SimplexNoiseAlpha(iaa.Add(10), name="SimplexNoiseAlpha"),
iaa.FrequencyNoiseAlpha(exponent=(-2, 2),
first=iaa.Add(10),
name="SimplexNoiseAlpha"),
iaa.Superpixels(p_replace=0.01, n_segments=64),
iaa.Resize({
"height": 4,
"width": 4
}, name="Resize"),
iaa.CropAndPad(px=(-10, 10), name="CropAndPad"),
iaa.Pad(px=(0, 10), name="Pad"),
iaa.Crop(px=(0, 10), name="Crop")
]
for aug in augs:
for (nb_channels, images_c) in images:
if aug.name != "Resize":
images_aug = aug.augment_images(images_c)
assert images_aug.shape == images_c.shape
image_aug = aug.augment_image(images_c[0])
assert image_aug.shape == images_c[0].shape
else:
images_aug = aug.augment_images(images_c)
image_aug = aug.augment_image(images_c[0])
if images_c.ndim == 3:
assert images_aug.shape == (4, 4, 4)
assert image_aug.shape == (4, 4)
else:
assert images_aug.shape == (4, 4, 4, images_c.shape[3])
assert image_aug.shape == (4, 4, images_c.shape[3])
box.append(int(float(xmlbox.find('xmin').text)))
box.append(int(float(xmlbox.find('ymin').text)))
box.append(int(float(xmlbox.find('xmax').text)))
box.append(int(float(xmlbox.find('ymax').text)))
box.append(cls_id)
boxes.append(box)
return np.array(boxes)
aug = iaa.SomeOf(
(0, 3),
[
iaa.GammaContrast([0.5, 1.5]),
iaa.Affine(translate_percent=[-0.05, 0.05],
scale=[0.9, 1.1],
mode='constant'), # mode='edge'
iaa.Fliplr(0.5),
iaa.Flipud(0.5)
])
# In[ ]:
class DataGenerator(keras.utils.Sequence):
# list_IDs: 图片名list xml_file list： 图片名对应的xml_file list
def __init__(self,
list_IDs,
list_xmls,
num_class,
cls2id,
def test_determinism():
reseed()
images = [
ia.quokka(size=(128, 128)),
ia.quokka(size=(64, 64)),
ia.imresize_single_image(skimage.data.astronaut(), (128, 256))
]
images.extend([ia.quokka(size=(16, 16))] * 20)
keypoints = [
ia.KeypointsOnImage([
ia.Keypoint(x=20, y=10),
ia.Keypoint(x=5, y=5),
ia.Keypoint(x=10, y=43)
],
shape=(50, 60, 3))
] * 20
augs = [
iaa.Sequential([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.SomeOf(1, [iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.OneOf([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.Sometimes(0.5, iaa.Fliplr(1.0)),
iaa.WithColorspace("HSV", children=iaa.Add((-50, 50))),
# iaa.WithChannels([0], iaa.Add((-50, 50))),
# iaa.Noop(name="Noop-nochange"),
# iaa.Lambda(
# func_images=lambda images, random_state, parents, hooks: images,
# func_keypoints=lambda keypoints_on_images, random_state, parents, hooks: keypoints_on_images,
# name="Lambda-nochange"
# ),
# iaa.AssertLambda(
# func_images=lambda images, random_state, parents, hooks: True,
# func_keypoints=lambda keypoints_on_images, random_state, parents, hooks: True,
# name="AssertLambda-nochange"
# ),
# iaa.AssertShape(
# (None, None, None, 3),
# check_keypoints=False,
# name="AssertShape-nochange"
# ),
iaa.Resize((0.5, 0.9)),
iaa.CropAndPad(px=(-50, 50)),
iaa.Pad(px=(1, 50)),
iaa.Crop(px=(1, 50)),
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Superpixels(p_replace=(0.25, 1.0), n_segments=(16, 128)),
# iaa.ChangeColorspace(to_colorspace="GRAY"),
iaa.Grayscale(alpha=(0.1, 1.0)),
iaa.GaussianBlur((0.1, 3.0)),
iaa.AverageBlur((3, 11)),
iaa.MedianBlur((3, 11)),
# iaa.Convolve(np.array([[0, 1, 0],
# [1, -4, 1],
# [0, 1, 0]])),
iaa.Sharpen(alpha=(0.1, 1.0), lightness=(0.8, 1.2)),
iaa.Emboss(alpha=(0.1, 1.0), strength=(0.8, 1.2)),
iaa.EdgeDetect(alpha=(0.1, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.1, 1.0), direction=(0.0, 1.0)),
iaa.Add((-50, 50)),
iaa.AddElementwise((-50, 50)),
iaa.AdditiveGaussianNoise(scale=(0.1, 1.0)),
iaa.Multiply((0.6, 1.4)),
iaa.MultiplyElementwise((0.6, 1.4)),
iaa.Dropout((0.3, 0.5)),
iaa.CoarseDropout((0.3, 0.5), size_percent=(0.05, 0.2)),
iaa.Invert(0.5),
iaa.ContrastNormalization((0.6, 1.4)),
iaa.Affine(scale=(0.7, 1.3),
translate_percent=(-0.1, 0.1),
rotate=(-20, 20),
shear=(-20, 20),
order=ia.ALL,
mode=ia.ALL,
cval=(0, 255)),
iaa.PiecewiseAffine(scale=(0.1, 0.3)),
iaa.ElasticTransformation(alpha=0.5)
]
augs_affect_geometry = [
iaa.Sequential([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.SomeOf(1, [iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.OneOf([iaa.Fliplr(0.5), iaa.Flipud(0.5)]),
iaa.Sometimes(0.5, iaa.Fliplr(1.0)),
iaa.Resize((0.5, 0.9)),
iaa.CropAndPad(px=(-50, 50)),
iaa.Pad(px=(1, 50)),
iaa.Crop(px=(1, 50)),
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Affine(scale=(0.7, 1.3),
translate_percent=(-0.1, 0.1),
rotate=(-20, 20),
shear=(-20, 20),
order=ia.ALL,
mode=ia.ALL,
cval=(0, 255)),
iaa.PiecewiseAffine(scale=(0.1, 0.3)),
iaa.ElasticTransformation(alpha=(5, 100), sigma=(3, 5))
]
for aug in augs:
aug_det = aug.to_deterministic()
images_aug1 = aug_det.augment_images(images)
images_aug2 = aug_det.augment_images(images)
aug_det = aug.to_deterministic()
images_aug3 = aug_det.augment_images(images)
images_aug4 = aug_det.augment_images(images)
assert array_equal_lists(images_aug1, images_aug2), \
"Images (1, 2) expected to be identical for %s" % (aug.name,)
assert array_equal_lists(images_aug3, images_aug4), \
"Images (3, 4) expected to be identical for %s" % (aug.name,)
assert not array_equal_lists(images_aug1, images_aug3), \
"Images (1, 3) expected to be different for %s" % (aug.name,)
for aug in augs_affect_geometry:
aug_det = aug.to_deterministic()
kps_aug1 = aug_det.augment_keypoints(keypoints)
kps_aug2 = aug_det.augment_keypoints(keypoints)
aug_det = aug.to_deterministic()
kps_aug3 = aug_det.augment_keypoints(keypoints)
kps_aug4 = aug_det.augment_keypoints(keypoints)
assert keypoints_equal(kps_aug1, kps_aug2), \
"Keypoints (1, 2) expected to be identical for %s" % (aug.name,)
assert keypoints_equal(kps_aug3, kps_aug4), \
"Keypoints (3, 4) expected to be identical for %s" % (aug.name,)
assert not keypoints_equal(kps_aug1, kps_aug3), \
"Keypoints (1, 3) expected to be different for %s" % (aug.name,)
def Augmentation(input_image):
with tf.Graph().as_default():
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
log_device_placement=False))
with sess.as_default():
pnet, rnet, onet = detect_face.create_mtcnn(sess, './npy')
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
aug_name = input_image.split("/")[-1].split(".")[0]
minsize = 35 # minimum size of face
threshold = [0.6, 0.7, 0.7] # three steps's threshold
factor = 0.709 # scale factor
margin = 44
image_size = 200
nb_batches = 16
aug_faces = []
batches = []
seq = iaa.Sequential(
[
iaa.Fliplr(0.5),
sometimes(
iaa.CropAndPad(
percent=(-0.05, 0.1), pad_mode=ia.ALL, pad_cval=(0, 255))),
sometimes(
iaa.Affine(scale={
"x": (0.8, 1.0),
"y": (0.8, 1.0)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (0, 0.2)
},
rotate=(-10, 10),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255))),
iaa.SomeOf(
(0, 4),
[
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 1.0)), # emboss images
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.2, 0.5)),
iaa.DirectedEdgeDetect(alpha=(0.2, 0.5),
direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.Add((-10, 10), per_channel=0.5),
iaa.AddToHueAndSaturation((-20, 20)),
iaa.ContrastNormalization((0.5, 1.5), per_channel=0.5),
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 2), sigma=0.25)),
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.03))),
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
],
random_order=True)
img = misc.imread(input_image)
if img.ndim < 2:
print("Unable !")
elif img.ndim == 2:
img = facenet.to_rgb(img)
img = img[:, :, 0:3]
batches.append(np.array([img for _ in range(nb_batches)], dtype=np.uint8))
aug_images = seq.augment_images(batches[0])
for aug_img in aug_images:
bounding_boxes, _ = detect_face.detect_face(aug_img, minsize, pnet,
rnet, onet, threshold,
factor)
nrof_faces = bounding_boxes.shape[0]
if nrof_faces > 0:
det = bounding_boxes[:, 0:4]
img_size = np.asarray(img.shape)[0:2]
if nrof_faces > 1:
bounding_box_size = (det[:, 2] - det[:, 0]) * (det[:, 3] -
det[:, 1])
img_center = img_size / 2
offsets = np.vstack([
(det[:, 0] + det[:, 2]) / 2 - img_center[1],
(det[:, 1] + det[:, 3]) / 2 - img_center[0]
])
offset_dist_squared = np.sum(np.power(offsets, 2.0), 0)
index = np.argmax(bounding_box_size -
offset_dist_squared * 2.0)
det = det[index, :]
det = np.squeeze(det)
bb_temp = np.zeros(4, dtype=np.int32)
bb_temp[0] = det[0]
bb_temp[1] = det[1]
bb_temp[2] = det[2]
bb_temp[3] = det[3]
cropped_temp = aug_img[bb_temp[1]:bb_temp[3],
bb_temp[0]:bb_temp[2], :]
scaled_temp = misc.imresize(cropped_temp, (image_size, image_size),
interp='bilinear')
aug_faces.append(scaled_temp)
return aug_faces
iaa.SomeOf(
(0, 3),
[
iaa.OneOf([
iaa.GaussianBlur(
(0,
2.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(
k=(1, 5)
), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(
k=(1, 5)
), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.03 * 255),
per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.01, 0.03),
per_channel=0.2),
]),
iaa.Add(
(-10, 10), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.ContrastNormalization(
(0.3, 1.0),
per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(
0.01, 0.05))), # sometimes move parts of the image around
],
random_order=True)
def train_aenet(section1_epochs=10,
section2_epochs=20,
section3_epochs=300,
learning_rate=0.01,
learning_momentum=0.9,
optimiser='Adam',
add_freq=0.1,
add_value=(-10, 10),
add_pc_freq=0.5,
multiply_freq=0.1,
multiply_value=(0.75, 1.25),
multiply_pc_freq=0.5,
snp_freq=0.1,
snp_p=0.05,
jpeg_freq=0.1,
jpeg_compression=(1, 5),
gaussian_freq=0.1,
gaussian_sigma=(0.01, 0.7),
motion_freq=0.1,
motion_k=(3, 10),
contrast_freq=0.1,
contrast_alpha=(0.5, 1.5),
fliplr=0.5,
flipud=0.5,
affine_freq=0.1,
affine_scale=(0, 0.02),
transform_freq=0.1,
transform_scale=(0, 0.05),
elastic_transformations=False,
elastic_freq=0.1,
elastic_sigma=(4, 6),
elastic_alpha=(0, 7),
rotate=1,
dataset="/scratch/jw22g14/FK2018/second_set/",
log_file="",
separate_channel_operations=0):
config = FKConfig()
config.display()
model = modellib.MaskRCNN(mode="training",
config=config,
model_dir=DEFAULT_LOGS_DIR + log_file)
model_path = COCO_MODEL_PATH
model.load_weights(model_path,
by_name=True,
exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox",
"mrcnn_mask"
])
dataset_train = FKDataset()
dataset_train.load_fk(dataset + "/dive1")
dataset_train.load_fk(dataset + "/dive3")
dataset_train.prepare()
# Validation dataset
dataset_val = FKDataset()
dataset_val.load_fk(dataset + "/dive2")
dataset_val.prepare()
# Image Augmentation
# Right/Left flip 50% of the time
augmentation = iaa.Sequential([
iaa.Sometimes(add_freq,
iaa.Add(value=add_value, per_channel=add_pc_freq)),
iaa.Sometimes(
multiply_freq,
iaa.Multiply(mul=multiply_value, per_channel=multiply_pc_freq)),
iaa.Sometimes(snp_freq, iaa.SaltAndPepper(snp_p)),
iaa.Sometimes(jpeg_freq,
iaa.JpegCompression(compression=jpeg_compression)),
iaa.Sometimes(gaussian_freq, iaa.GaussianBlur(sigma=gaussian_sigma)),
iaa.Sometimes(motion_freq, iaa.MotionBlur(k=motion_k)),
iaa.Sometimes(contrast_freq, iaa.LinearContrast(alpha=contrast_alpha)),
iaa.Fliplr(fliplr),
iaa.Flipud(flipud),
iaa.Sometimes(
affine_freq,
iaa.PiecewiseAffine(scale=affine_scale,
nb_rows=8,
nb_cols=8,
polygon_recoverer='auto')),
iaa.Sometimes(
transform_freq,
iaa.PerspectiveTransform(scale=transform_scale, keep_size=True)),
iaa.Sometimes(
elastic_freq,
iaa.ElasticTransformation(sigma=elastic_sigma,
alpha=elastic_alpha)),
iaa.Sometimes(rotate, iaa.Rot90([0, 1, 2, 3]))
],
random_order=True)
auglist = [
#iaa.Add(value=add_value, per_channel=separate_channel_operations),
#iaa.Multiply(mul=multiply_value, per_channel=separate_channel_operations),
#iaa.SaltAndPepper(snp_p),
#iaa.JpegCompression(compression=jpeg_compression),
#iaa.GaussianBlur(sigma=gaussian_sigma),
#iaa.MotionBlur(k=motion_k),
#iaa.LinearContrast(alpha=contrast_alpha),
#iaa.Fliplr(fliplr),
#iaa.Flipud(flipud),
iaa.PiecewiseAffine(scale=affine_scale,
nb_rows=8,
nb_cols=8,
polygon_recoverer='auto'),
iaa.PerspectiveTransform(scale=transform_scale, keep_size=True),
#iaa.Rot90([0,1,2,3]),
]
if (elastic_transformations):
auglist.append(
iaa.ElasticTransformation(sigma=elastic_sigma,
alpha=elastic_alpha))
augmentation = iaa.SomeOf((0, 5), auglist, random_order=True)
print("Training RPN")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate,
epochs=5, #40
layers='rpn',
augmentation=augmentation)
print("Training network heads")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate,
epochs=section1_epochs, #40
layers='heads',
augmentation=augmentation)
# Training - Stage 2
# Finetune layers from ResNet stage 4 and up
print("Fine tune Resnet stage 4 and up")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate,
epochs=section2_epochs, #120
layers='4+',
augmentation=augmentation)
# Training - Stage 3
# Fine tune all layers
print("Fine tune all layers")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate / 10,
epochs=section3_epochs, #160
layers='all',
augmentation=augmentation,
)
def __init__(
self,
inputdata,
inputlabels,
augs="basic", #["all","basic","form","valalt","pxlalt","imgalt"]
num_outs=5,
og_out=True,
mode='G',
em=0,
intensity=1.0,
rescaledata=None,
formatd='NCHW',
min_augs=0,
max_augs=5):
if self.mode.lower() == 'g':
self.NM = self.rung
elif self.mode.lower() == 'i':
self.NM = self.runi()
elif self.mode.lower() == 'i2':
self.NM = self.runi2()
else:
print(
"invalid mode, use 'g' for generator or 'i' for iterator or 'i2'"
)
exit()
self.minaug = min_augs
self.maxaug = max_augs
#self.affineopt=["scale","translate_percent","translate_px","rotate","shear"]
#self.chnlopt=[{"per_channel":True},{"per_channel":False}]
if len(inputdata.shape) == 4:
self.D = 4
elif len(inputdata.shape) == 3:
self.D = 3
elif len(inputdata.shape) == 2:
self.D = 2
if formatd == "NCHW":
if self.D == 4:
self.inputd = np.transpose(inputdata, [0, 2, 3, 1])
elif self.D == 3:
self.inputd = np.transpose(inputdata, [1, 2, 0])
else:
self.inputd = inputdata
self.Y = inputlabels
leninten = 8
if isinstance(intensity, (float, int)):
itensity = [intensity for _ in range(leninten)]
else:
assert len(intensity) == leninten
self.datashape = np.array(inputdata.shape) #inputdata[0].shape
if self.datashape.min() == self.datashape[-1]:
self.pixls = self.datashape[:-1]
elif self.datashape.min() == self.datashape[1]:
self.pixls = np.delete(self.datashape, 1)
elif self.datashape.shape == (3, ):
self.pixls = self.datashape[1:]
else:
print("error cannot fin the shape of images")
exit()
# can use "keep-aspect-ratio" for an arg to have a relative and absolute scale
#or can also use list for randomization between options
self.scalevals = (0.5 / (2 * intensity), 1.0) #use % of image
self.augs = augs
self.Pchances = 0.44 * itensity[0]
self.intrange = ((ceil(10 * intensity[1]),
ceil(10 + 140 * itensity[1])))
self.windowrange = (ceil(2 * intensity[2]),
ceil((min(self.pixls) / 5) - 8) * intensity[2]
) #mean/median things
self.relatrange = (0.1 * intensity[3], 0.95 * intensity[3]
) #normalisation,invert
self.bigfloat = (
0.085 * intensity[4], 1.75 * intensity[4]
) #some scale values,multiply,contrastnorm,elasti trans,(sigman&alpha)
self.smallfloat = (0.001 * intensity[5], 0.45 * intensity[5]
) #coarse dropout/droput(p)
self.addrange = (ceil(-140 * intensity[6]),
ceil(140 * intensity[6]))
self.multrange = (-2.0 * intensity[7], 2.0 * intensity[7])
self.perchannelsplit = 0.75 * intensity[
8] #used for per_channel on the mult
self.allaugs = {
"add":
IAGA.Add(value=self.addrange, per_channel=0.75 * intensity),
"scale":
IAGA.Scale(size=self.scalevals),
"adde":
IAGA.AddElementwise(value=self.addrange,
per_channel=0.75 * intensity),
"addg":
IAGA.AdditiveGaussianNoise(scale=(0, self.smallfloat[1] * 255),
per_channel=0.75 * intensity),
"addh":
IAGA.AddToHueAndSaturation(value=self.addrange,
per_channel=0.75 * intensity),
"mult":
IAGA.Multiply(mul=self.bigfloat, per_channel=0.75 * intensity),
"mule":
IAGA.MultiplyElementwise(mul=self.bigfloat,
per_channel=0.75 * intensity),
"drop":
IAGA.Dropout(p=self.smallfloat, per_channel=0.75 * intensity),
"cdrop":
IAGA.CoarseDropout(p=self.smallfloat,
size_px=None,
size_percent=self.smallfloat,
per_channel=True,
min_size=3),
"inv":
IAGA.Invert(p=self.Pchances,
per_channel=0.75 * intensity,
min_value=-255,
max_value=255),
"cont":
IAGA.ContrastNormalization(alpha=self.bigfloat,
per_channel=0.75 * intensity),
"aff":
IAGA.Affine(
scale=self.bigfloat,
translate_percent={
'x': (-40 * intensity, 40 * intensity),
'y': (-40 * intensity, 40 * intensity)
},
translate_px=None, #moving functions
rotate=(-360 * intensity, 360 * intensity),
shear=(-360 * intensity, 360 * intensity),
order=[0, 1] #2,3,4,5 may be too much
,
cval=0, #for filling
mode=["constant", "edge", "reflect", "symmetric",
"wrap"][em], #filling method
deterministic=False,
random_state=None),
"paff":
IAGA.PiecewiseAffine(
scale=(-0.075 * intensity, 0.075 * intensity),
nb_rows=(ceil(2 * intensity), ceil(7 * intensity)),
nb_cols=(ceil(2 * intensity), ceil(7 * intensity)),
order=[0, 1],
cval=0,
mode=["constant", "edge", "reflect", "symmetric",
"wrap"][em],
deterministic=False,
random_state=None),
"elas":
IAGA.ElasticTransformation(alpha=self.bigfloat,
sigma=self.relatrange),
"noop":
IAGA.Noop(name="nope"),
#IAGA.Lambda:{},
"cropad":
IAGA.CropAndPad(
px=None,
percent=(-0.65 * intensity[7], 0.65 * intensity[7]),
pad_mode=[
"constant", "edge", "reflect", "symmetric", "wrap"
][em],
pad_cval=0,
keep_size=True,
sample_independently=True,
),
"fliplr":
IAGA.Fliplr(p=self.Pchances),
"flipud":
IAGA.Flipud(p=self.Pchances),
"spixel":
IAGA.Superpixels(p_replace=self.Pchances,
n_segments=self.intrange),
#IAGA.ChangeColorspace:,
"gray":
IAGA.Grayscale(alpha=self.relatrange),
"gblur":
IAGA.GaussianBlur(sigma=self.bigfloat),
"ablur":
IAGA.AverageBlur(k=self.windowrange),
"mblur":
IAGA.MedianBlur(k=self.windowrange),
#IAGA.BilateralBlur,
#IAGA.Convolve:,
"sharp":
IAGA.Sharpen(alpha=self.relatrange, lightness=self.bigfloat),
"embo":
IAGA.Emboss(alpha=self.relatrange, strenght=self.bigfloat),
"edge":
IAGA.EdgeDetect(alpha=self.relatrange),
"dedge":
IAGA.DirectedEdgeDetect(alpha=self.bigfloat,
direction=(-1.0 * intensity,
1.0 * intensity)),
"pert":
IAGA.PerspectiveTransform(scale=self.smallfloat),
"salt":
IAGA.Salt(p=self.Pchances, per_channel=0.75 * intensity),
#IAGA.CoarseSalt(p=, size_px=None, size_percent=None,per_channel=False, min_size=4),
#IAGA.CoarsePepper(p=, size_px=None, size_percent=None,"per_channel=False, min_size=4),
#IAGA.CoarseSaltAndPepper(p=, size_px=None, size_percent=None,per_channel=False, min_size=4),
"pep":
IAGA.Pepper(p=self.Pchances, per_channel=0.75 * intensity),
"salpep":
IAGA.SaltAndPepper(p=self.Pchances,
per_channel=0.75 * intensity),
#"alph":IAGA.Alpha(factor=,first=,second=,per_channel=0.75*intensity,),
#"aplhe":IAGA.AlphaElementwise(factor=,first=,second=,per_channel=0.75*intensity,),
#IAGA.FrequencyNoiseAlpha(exponent=(-4, 4),first=None, second=None, per_channel=False,size_px_max=(4, 16), upscale_method=None,iterations=(1, 3), aggregation_method=["avg", "max"],sigmoid=0.5, sigmoid_thresh=None,),
#IAGA.SimplexNoiseAlpha(first=None, second=None, per_channel=False,size_px_max=(2, 16), upscale_method=None,iterations=(1, 3), aggregation_method="max",sigmoid=True, sigmoid_thresh=None,),
}
["all", "basic", "form", "valalt", "pxlalt", "imgalt"]
self.augs = []
if (augs == "all") or ("all" in augs):
self.augs = [
"add",
"scale",
"adde",
"addg",
"addh",
"mult",
"mule",
"drop",
"cdrop",
"inv",
"cont",
"aff",
"paff",
"elas",
"noop",
"cropad",
"fliplr",
"flipud",
"spixel",
"gray",
"gblur",
"ablur",
"mblur",
"sharp",
"embo",
"edge",
"dedge",
"pert",
"salt",
"pep",
"salpep",
] #"alph", "aplhe",]
else:
if (augs == "basic") or ("basic" in augs):
self.augs.append([
"add", "scale", "addh", "mult", "drop", "cont", "noop"
])
if (augs == "form") or ("form" in augs):
self.augs + [
"scale", "aff", "paff", "elas", "noop", "pert"
]
if (augs == "valalt") or ("valalt" in augs):
self.augs + [
"mult", "mule", "inv", "fliplr", "flipud", "cropad",
"noop"
]
if (augs == "pxlalt") or ("pxlalt" in augs):
self.augs + [
"addg", "drop", "salt", "pep", "salpep", "noop"
]
if (augs == "imgalt") or ("imgalt" in augs):
self.augs + [
"elas",
"noop",
"spixel",
"gblur",
"ablur",
"mblur",
"sharp",
"embo",
"edge",
"dedge",
]
if len(augs) == 0:
self.augs + [
"add",
"scale",
"addh",
"drop",
"cont",
"aff",
"elas",
"noop",
"cropad",
"gray",
"ablur",
"sharp",
"salpep",
]
self.AUG = IAGA.SomeOf((self.minaug, self.maxaug),
self.augs,
random_order=True)
"""self.affineopts={"scale":self.biglfoat,
"translate_percent":{'x':(-40*intensity,40*intensity),'y':(-40*intensity,40*intensity)}, "translate_px":None,#moving functions
"rotate":(-360*intensity,360*intensity), "shear":(0*intensity,360*intensity),
"order":[0,1]#2,3,4,5 may be too much
, "cval":0,#for filling
"mode":"constant",#filling method
"deterministic":False,
"random_state":None}
self.pieceaffinev={"scale"=(-0.075*intensity,0.075*intensity), "nb_rows"=(ceil(2*intensity),ceil(7*intensity)), "nb_cols"=(ceil(2*intensity),ceil(7*intensity)),
"order"=[0,1], "cval"=0, "mode"="constant",
"deterministic"=False, "random_state"=None}"""
self.num_outs = num_outs - og_out
self.og_out = og_out
self.mode = mode
self.iimg = -1
self.iout = 0
try:
self.len = inputdata.shape[0]
except:
self.len = len(inputdata)
def __iter__(self):
return self
def __next__(self):
return (self.NM())
def next(self):
return (self.NM())
def runi(self):
if self.iimg == self.len:
raise StopIteration
self.iimg += 1
img = self.inputd[self.iimg]
y = self.Y[self.iimg]
out = np.broadcast_to(img, (self.num_out, *img.shape[-3:]))
out = self.AUG.augment_images(out[self.og_out:])
if self.og_out:
if len(img.shape) == 3:
out = np.concatenate(out, np.expand_dims(img, 0))
else:
out = np.concatenate(out, img)
if self.format == "NCHW":
out = np.transpose(out, [0, 3, 1, 2])
return ([(outi, y) for outi in out])
def runi2(self):
if self.iimg == self.len:
raise StopIteration
if (self.iout == self.num_outs) or (self.iimg == -1):
self.iimg += 1
self.iout = 0
img = self.inputd[self.iimg]
y = self.Y[self.iimg]
out = np.broadcast_to(img, (self.num_out, *img.shape[-3:]))
self.out = self.AUG.augment_images(out[self.og_out:])
if self.og_out:
if len(img.shape) == 3:
self.out = np.concatenate(out,
np.expand_dims(img, 0))
else:
self.out = np.concatenate(out, img)
if self.format == "NCHW":
self.out = np.transpose(out, [0, 3, 1, 2])
outp = (self.out[self.iout], y)
else:
self.iout += 1
outp = (self.out[self.iout], self.Y[self.iimg])
return (outp)
def rung(self):
for ix, img in enumerate(self.inputd):
out = np.broadcast_to(img, (self.num_out, img.shape[-3:]))
out = self.AUG.augment_images(out[self.og_out:])
y = self.Y[ix]
if self.og_out:
if len(img.shape) == 3:
out = np.concatenate(out, np.expand_dims(img, 0))
else:
out = np.concatenate(out, img)
if self.format == "NCHW":
out = (np.transpose(out, [0, 3, 1, 2]))
for sout in out:
yield (sout, y)
def train_nnet(section1_epochs=10,
section2_epochs=20,
section3_epochs=300,
learning_rate=0.01,
learning_momentum=0.9,
steps_per_epoch=50,
batch_size=1,
optimiser='Adam',
add_freq=0.1,
add_value=(-5, 5),
multiply_value=(0.9, 1.1),
snp_p=0.03,
jpeg_compression=(1, 5),
gaussian_sigma=(0.01, 0.5),
motion_k=(3, 10),
contrast_alpha=(0.5, 1.5),
fliplr=0.5,
flipud=0.5,
affine_scale=(0, 0.02),
transform_scale=(0, 0.05),
elastic_transformations=False,
flip_rotate=False,
extras=False,
elastic_sigma=(4, 6),
elastic_alpha=(0, 7),
train_dataset="/scratch/jw22g14/FK2018/second_set/",
val_dataset="",
log_file="",
separate_channel_operations=0,
mean_pixel=np.array([75, 75, 75]),
std_pixel=np.array([25, 25, 25])):
config = FKConfig(learning_rate=learning_rate,
learning_momentum=learning_momentum,
steps_per_epoch=steps_per_epoch,
batch_size=batch_size,
mean_pixel=mean_pixel,
std_pixel=std_pixel)
model = modellib.MaskRCNN(mode="training",
config=config,
model_dir=DEFAULT_LOGS_DIR + log_file)
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# config = tf.config.experimental.set_memory_growth(physical_devices[0], True)
# with tf.InteractiveSession(config=config).as_default():
model_path = COCO_MODEL_PATH
model.load_weights(model_path,
by_name=True,
exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox",
"mrcnn_mask"
])
dataset_train = FKDataset()
dataset_train.load_fk(train_dataset)
dataset_train.prepare()
# Validation dataset
dataset_val = FKDataset()
dataset_val.load_fk(val_dataset)
dataset_val.prepare()
auglist = [
iaa.Add(value=add_value, per_channel=separate_channel_operations),
iaa.Multiply(mul=multiply_value,
per_channel=separate_channel_operations),
iaa.SaltAndPepper(snp_p),
iaa.JpegCompression(compression=jpeg_compression),
iaa.GaussianBlur(sigma=gaussian_sigma),
iaa.MotionBlur(k=motion_k),
iaa.LinearContrast(alpha=contrast_alpha),
]
auglist = [
iaa.Flipud(1),
iaa.Fliplr(1),
iaa.Rot90([1, 2, 3]),
iaa.GaussianBlur(sigma=(1, 2)),
iaa.JpegCompression(compression=(25, 50))
]
if (elastic_transformations):
auglist.extend([
iaa.ElasticTransformation(sigma=elastic_sigma,
alpha=elastic_alpha),
iaa.PiecewiseAffine(scale=affine_scale,
nb_rows=8,
nb_cols=8,
polygon_recoverer='auto'),
iaa.PerspectiveTransform(scale=transform_scale, keep_size=True),
])
if (extras):
auglist.extend([
iaa.Add(value=add_value, per_channel=separate_channel_operations),
iaa.Multiply(mul=multiply_value,
per_channel=separate_channel_operations),
iaa.SaltAndPepper(snp_p),
iaa.MotionBlur(k=motion_k),
iaa.LinearContrast(alpha=contrast_alpha),
])
augmentation = iaa.SomeOf(
(0, len(auglist)), auglist, random_order=True
) #originally 0, 5, trying smaller number for fewer_augs run
print("Training RPN")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate,
epochs=5, #40
layers='rpn',
augmentation=augmentation)
print("Training network heads")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate,
epochs=section1_epochs, #40
layers='heads',
augmentation=augmentation)
# Training - Stage 2
# Finetune layers from ResNet stage 4 and up
print("Fine tune Resnet stage 4 and up")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate,
epochs=section2_epochs, #120
layers='4+',
augmentation=augmentation)
# Training - Stage 3
# Fine tune all layers
print("Fine tune all layers")
model.train(
dataset_train,
dataset_val,
learning_rate=learning_rate / 10,
epochs=section3_epochs, #160
layers='all',
augmentation=augmentation,
)
def __init__(self,
input_path,
batch_size,
num_samples_in_h5,
sequence_length,
labels,
stride=1,
is_debug=False,
is_validation=False,
is_yolo_feats=True,
is_augment=False):
self.input_path = input_path
self.batch_size = batch_size
self.num_in_h5 = num_samples_in_h5
self.sequence_length = sequence_length
self.labels = labels
self.stride = stride
self.is_debug = is_debug
self.is_validation = is_validation
self.is_yolo_feats = is_yolo_feats
assert (self.num_in_h5 % self.batch_size == 0)
self.num_batches_in_h5 = self.num_in_h5 / self.batch_size
filepaths = os.listdir(self.input_path)
shuffle(filepaths)
if not is_validation:
self.filepaths = filepaths[:-3]
else:
self.filepaths = filepaths[-3:]
self.is_augment = is_augment
sometimes = lambda aug: iaa.Sometimes(0.35, aug)
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
self.aug_pipe = iaa.Sequential(
[
# apply the following augmenters to most images
# iaa.Fliplr(0.5), # horizontally flip 50% of all images
#iaa.Flipud(0.2), # vertically flip 20% of all images
# sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
# sometimes(iaa.Affine(
# scale={"x": (0.85, 1.15), "y": (0.85, 1.15)}, # scale images to 80-120% of their size, individually per axis
# translate_percent={"x": (-0.15, 0.15), "y": (-0.15, 0.15)}, # translate by -20 to +20 percent (per axis)
# rotate=(-5, 5), # rotate by -45 to +45 degrees
#shear=(-5, 5), # shear by -16 to +16 degrees
#order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
#cval=(0, 255), # if mode is constant, use a cval between 0 and 255
#mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
# )),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 5),
[
#sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 2.0)
), # blur images with a sigma between 0 and 3.0
# iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
# iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
]),
# iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
#iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.04 * 255),
per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.05), per_channel=0.5
), # randomly remove up to 10% of the pixels
]),
iaa.Invert(0.05,
per_channel=True), # invert color channels
iaa.Add(
(-7, 7), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.Multiply(
(0.8, 1.25), per_channel=0.5
), # change brightness of images (50-150% of original value)
iaa.ContrastNormalization(
(0.8, 1.25),
per_channel=0.5), # improve or worsen the contrast
#iaa.Grayscale(alpha=(0.0, 1.0)),
],
random_order=True)
],
random_order=True)
iaa.SomeOf(
(0, 5),
[
sometimes(
iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))
), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0,
1.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(
k=(3, 5)
), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(
k=(3, 5)
), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.9, 1.1)), # sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.01 * 255),
per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.05), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.01, 0.03),
size_percent=(0.01, 0.02),
per_channel=0.2),
]),
iaa.Invert(0.01, per_channel=True), # invert color channels
iaa.Add(
(-2, 2), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation(
(-1, 1)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.9, 1.1), per_channel=0.5),
iaa.FrequencyNoiseAlpha(exponent=(-1, 0),
first=iaa.Multiply(
(0.9, 1.1), per_channel=True),
second=iaa.ContrastNormalization(
(0.9, 1.1)))
]),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(
0.01, 0.05))), # sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
iaa.SomeOf(
(0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
sometimes(
iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
# iaa.OneOf([
# iaa.GaussianBlur((0, 3.0)),
# iaa.AverageBlur(k=(2, 7)),
# iaa.MedianBlur(k=(3, 11)),
# ]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
# iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
# iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
# sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(
# alpha=(0, 0.7), direction=(0.0, 1.0)
# ),
# ])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
# iaa.Invert(0.05, per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
# iaa.Add((-10, 10), per_channel=0.5),
iaa.Add((0, 30)),
# Change brightness of images (50-150% of original value).
# iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.Multiply((1.0, 1.2)),
# Improve or worsen the contrast of images.
# iaa.LinearContrast((0.5, 2.0), per_channel=0.5),
iaa.LinearContrast((1.0, 1.5)),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(iaa.ElasticTransformation(alpha=(0, 2.5), sigma=0.25)
),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.1)))
],
# do all of the above augmentations in random order
random_order=True)
def draw_single_sequential_images():
image = misc.imresize(ndimage.imread("quokka.jpg")[0:643, 0:643], (128, 128))
"""
rarely = lambda aug: iaa.Sometimes(0.1, aug)
sometimes = lambda aug: iaa.Sometimes(0.25, aug)
often = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.5), # vertically flip 50% of all images
rarely(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
often(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
rarely(iaa.GaussianBlur((0, 3.0))), # blur images with a sigma between 0 and 3.0
rarely(iaa.AverageBlur(k=(2, 7))), # blur image using local means with kernel sizes between 2 and 7
rarely(iaa.MedianBlur(k=(3, 11))), # blur image using local medians with kernel sizes between 2 and 7
sometimes(iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5))), # sharpen images
sometimes(iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0))), # emboss images
# search either for all edges or for directed edges
rarely(iaa.Sometimes(0.5,
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
)),
often(iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5)), # add gaussian noise to images
often(iaa.Dropout((0.0, 0.1), per_channel=0.5)), # randomly remove up to 10% of the pixels
often(iaa.CoarseDropout((0.0, 0.05), size_percent=(0.02, 0.25), per_channel=0.5)),
rarely(iaa.Invert(0.25, per_channel=True)), # invert color channels
often(iaa.Add((-10, 10), per_channel=0.5)), # change brightness of images (by -10 to 10 of original value)
often(iaa.Multiply((0.5, 1.5), per_channel=0.5)), # change brightness of images (50-150% of original value)
often(iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5)), # improve or worsen the contrast
sometimes(iaa.Grayscale(alpha=(0.0, 1.0))),
often(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
rarely(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)) # move pixels locally around (with random strengths)
],
random_order=True
)
"""
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
sometimes(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges
sometimes(iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
iaa.Multiply((0.5, 1.5), per_channel=0.5), # change brightness of images (50-150% of original value)
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
],
random_order=True
)
],
random_order=True
)
grid = seq.draw_grid(image, cols=8, rows=8)
misc.imsave("examples_grid.jpg", grid)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
iaa.OneOf([
iaa.GaussianBlur((0, 1.0)), # blur images with a sigma between 0 and 3.0
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
#iaa.SimplexNoiseAlpha(iaa.OneOf([
#iaa.EdgeDetect(alpha=(0.5, 1.0)),
#])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.08*255)), # add gaussian noise to images, for 50% of the overall images.
iaa.OneOf([
iaa.Dropout((0.01, 0.1)), # randomly remove up to 10% of the pixels
]),
iaa.Add((-10, 10)), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-10, 10)), # change hue and saturation
# either change the brightness of the whole image (sometimes per channel) or change the brightness of subareas
iaa.ContrastNormalization((0.5, 2.0)), # improve or worsen the contrast
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True
)
],
random_order=True
def make_synthetic_image(self, file_base, added_images):
img_base = cv2.imread(join(self.bg_path, file_base))
# Store mask labels for later training, i.e. stores the corresponding object label for every mask channel
save_name = ''
mask_labels = []
mask_overlayed = np.zeros((img_base.shape[0], img_base.shape[1]),
dtype=np.uint8)[:, :, None]
# Get rid of placeholder channel entry
mask_overlayed = mask_overlayed[:, :, 1:]
if len(mask_overlayed.shape) < 3:
mask_overlayed = mask_overlayed[:, :, np.newaxis]
img_overlayed = copy(img_base)
# Perturb background
scale = np.random.uniform(0.6, 1.1)
img_perturbed = copy(img_overlayed)
img_perturbed = (img_perturbed * scale).astype(np.uint8)
img_perturbed[np.where(img_perturbed > 255)] = 255
img_perturbed[np.where(img_perturbed < 0)] = 0
img_overlayed = img_perturbed
for i, file_added in enumerate(added_images):
# Read image to be added on top
print("Added image: ", file_added)
img_added = cv2.imread(
join(self.mask_root_path,
file_added.split('_')[0], file_added))
if file_base.endswith('.jpg'):
mask_added = np.load(
join(self.mask_root_path,
file_added.split('_')[0],
file_added.split('.jpg')[0] + '.npy'))
else:
mask_added = np.load(
join(self.mask_root_path,
file_added.split('_')[0],
file_added.split('.png')[0] + '.npy'))
mask_labels.append(self.mask_root_path.split('/')[-1])
# Mask image
# img_added_masked = img_added * mask_added[:,:,np.newaxis]
aff = iaa.Sequential([
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
iaa.Affine(scale={
"x": (0.7, 1.3),
"y": (0.7, 1.3)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-90, 90),
shear=(-15, 15))
])
# Affine transform
aff_det = aff.to_deterministic()
img_added = aff_det.augment_image(img_added)
mask_added = aff_det.augment_image(mask_added)
# Static Transform
st = iaa.SomeOf(
(0, 3),
[
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.02 * 255), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.8, 1.2), per_channel=0.0),
# Improve or worsen the contrast of images.
iaa.ContrastNormalization((0.8, 1.2), per_channel=0.5),
iaa.OneOf([
iaa.Dropout((0.005, 0.01), per_channel=0.0),
iaa.CoarseDropout((0.01, 0.05),
size_percent=(0.01, 0.05),
per_channel=0.0),
]),
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 30), per_channel=0.0),
])
img_added = st.augment_image(img_added)
img_added_masked = img_added * mask_added[:, :, np.newaxis]
# # Augment masks
# img_added_masked, mask_added = self.translate_mask(img_added_masked, mask_added, \
# row_shift=randint(-MAX_SHIFT_ROW, MAX_SHIFT_ROW), \
# col_shift=randint(-MAX_SHIFT_COL, MAX_SHIFT_COL))
# img_added_masked, mask_added = self.rotate_mask(img_added_masked, mask_added, \
# angle=randint(-180,180,1), center=None, \
# scale=np.random.uniform(0.9, 1.1))
# img_added_masked, mask_added = self.perturb_intensity(img_added_masked, mask_added, scale=np.random.uniform(0.9,1.1))
# Apply masks
img_overlayed[np.where(
mask_added == 1)] = img_added_masked[np.where(mask_added == 1)]
for j in range(mask_overlayed.shape[-1]):
mask_overlayed[:, :, j] *= np.logical_not(mask_added)
mask_overlayed = np.concatenate([mask_overlayed, \
mask_added[:, :, np.newaxis]], axis=2)
# Save image and mask
if i > 0: connector = '_'
else: connector = ''
if file_base.endswith('.jpg'):
save_name += connector + file_added.split('.jpg')[0]
else:
save_name += connector + file_added.split('.png')[0]
# if same overlay combo exists, assign unique suffix
save_name += '-0'
while os.path.exists(join(self.save_path, save_name + '.jpg')):
index = int(save_name.split('-')[-1][0])
save_name = save_name.split('-')[0] + '-' + str(index + 1)
return img_overlayed, mask_overlayed, mask_labels, save_name
def custom_augmenter_v1(sometimes):
seq = iaa.Sequential(
[
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 4),
[
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(100, 200))
), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0
)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(
k=(2, 7)
), # blur image using local means with kernel sizes between 2 and 7
# iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
iaa.MotionBlur(k=(3, 7)
) # blur image using motion blur
# with angle between [-90, 90] and kernel size between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 2.0)), # emboss images
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255),
per_channel=0.25), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
]),
iaa.Add(
(-10, 10), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation(
(-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0)))
]),
iaa.ContrastNormalization(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 2.5), sigma=0.25)
), # move pixels locally around (with random strengths)
],
random_order=True)
],
random_order=True)
return seq
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = Deep_Lesion_Dataset()
############################## We need to separate the training and validation
### There is a variable in the dict, which is training_val_testing
### It's in image_info file,
# under annotations, image
#annotations['images'][0]['Train_Val_Test']
### TRAIN = 1
### VALIDATION = 2
### TEST = 3
###### NEED TO EXRACT THREE DIFFERENT DATASETS based on this.
## look at how "train"/"valid" is fed in below
dataset_train.load_deep_lesion("blah", "train")
dataset_train.prepare()
# Validation dataset
dataset_val = Deep_Lesion_Dataset()
dataset_val.load_deep_lesion("blah", "val")
dataset_val.prepare()
augmentation = iaa.SomeOf(
(0, 1),
[
#iaa.Fliplr(0.5),
iaa.Affine(scale={
"x": (0.7, 1.3),
"y": (0.7, 1.3)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-25, 25),
shear=(-2, 2)),
iaa.Multiply((0.7, 1.3))
])
########################################################################################
from keras.callbacks import ReduceLROnPlateau
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.3,
patience=patience1,
verbose=1,
mode='min',
cooldown=0,
min_lr=0)
callbacks = [reduce_lr]
print("Training Images: {}\nClasses: {}".format(
len(dataset_train.image_ids), dataset_train.class_names))
print("Validations Images: {}\nClasses: {}".format(
len(dataset_val.image_ids), dataset_val.class_names))
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
print("Training network heads")
model.train(
dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
#epochs=30,
#layers='heads')
epochs=20,
layers='heads',
custom_callbacks=callbacks, #############
augmentation=augmentation) ######################
print("finished training network")
