def __init__(self,
images,
config,
shuffle=True,
jitter=True,
norm=None,
flipflop=True,
shoechanger=True,
zeropad=True):
self.generator = None
self.flipflop = flipflop
self.shoechanger = shoechanger
if self.flipflop or self.shoechanger:
self.badshoes = []
for im in os.listdir('imgs/more_badshoes'):
self.badshoes.append(cv2.imread('imgs/more_badshoes/' + im))
self.zeropad = zeropad
self.images = images
self.config = config
self.shuffle = shuffle
self.jitter = jitter
self.norm = norm
self.anchors = [
BoundBox(0, 0, config['ANCHORS'][2 * i],
config['ANCHORS'][2 * i + 1])
for i in range(int(len(config['ANCHORS']) // 2))
]
### augmentors by https://github.com/aleju/imgaug
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
self.aug_pipe = iaa.Sequential(
[
# apply the following augmenters to most images
# iaa.Fliplr(0.5), # horizontally flip 50% of all images
# iaa.Flipud(0.2), # vertically flip 20% of all images
# sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
sometimes(
iaa.Affine(
# scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
# translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
# rotate=(-5, 5), # rotate by -45 to +45 degrees
# shear=(-5, 5), # shear by -16 to +16 degrees
# order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 5),
[
# sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0)
), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)),
# blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)),
# blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
# iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges
# sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
# ])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
# iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
# iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5),
# change brightness of images (by -10 to 10 of original value)
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# change brightness of images (50-150% of original value)
iaa.ContrastNormalization(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
# iaa.Grayscale(alpha=(0.0, 1.0)),
# sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
# sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
],
random_order=True)
],
random_order=True)
if shuffle:
np.random.shuffle(self.images)
def __init__(self, image):
self.img = image
# 随机通道处理,加减100以内
# self.aug_WithChannels = iaa.WithChannels((0,2), iaa.Add((-100, 100)))
# 随机裁剪和填充，percent为裁剪与填充比例，负数为放大后裁剪，正数为缩小和填充，pad_mode为填充方式，pad_cval为当空白填充时，填充像素值
self.aug_CropAndPad = iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode=ia.ALL,
pad_cval=(0, 255))
# 随机水平翻转,参数为概率
self.aug_Fliplr = iaa.Fliplr(0.5)
# 随机垂直翻转,参数为概率
self.aug_Flipud = iaa.Flipud(0.5)
# 超像素表示,p_replace被超像素代替的百分比,n_segments分割块数
self.aug_Superpixels = iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))
# 灰度化 (0.0,1.0),前者为偏彩色部分，后者为偏灰度部分，随机灰度化
self.aug_GrayScale = iaa.Grayscale(alpha=(0.0, 0.6))
# 高斯模糊
self.aug_GaussianBlur = iaa.GaussianBlur(sigma=(0, 3.0))
# 均值模糊，k为kernel size
self.aug_AverageBlur = iaa.AverageBlur(k=(2, 7))
# 中值模糊, k为kernel size
self.aug_MedianBlur = iaa.MedianBlur(k=(3, 11))
# 双边滤波,d为kernel size,sigma_color为颜色域标准差,sigma_space为空间域标准差
self.aug_BilateralBlur = iaa.BilateralBlur(sigma_color=(0, 250),
sigma_space=(0, 250),
d=(3, 7))
# 锐化
self.aug_Sharpen = iaa.Sharpen(alpha=(0.0, 1.0), lightness=(0.75, 2.0))
# 浮雕效果
self.aug_Emboss = iaa.Emboss(alpha=(0.0, 1.0), strength=(0.0, 1.5))
# 边缘检测
self.aug_EdgeDetect = iaa.EdgeDetect(alpha=(0.0, 1.0))
# 方向性边缘检测
self.aug_DirectedEdgeDetece = iaa.DirectedEdgeDetect(alpha=(0.0, 1.0),
direction=(0.0,
1.0))
# 暴力叠加像素值,每个像素统一加一个值
self.aug_Add = iaa.Add((-40, 40))
# 暴力叠加像素值，每个像素加不同的值
self.aug_AddElementwise = iaa.AddElementwise((-40, 40))
# 随机高斯加性噪声
self.aug_AdditiveGaussianNoise = iaa.AdditiveGaussianNoise(scale=(0.0,
0.1 *
255))
# 暴力乘法,每个像素统一乘以一个值
self.aug_Multiply = iaa.Multiply((0.8, 1.2))
# 暴力乘法,每个像素乘以不同值
self.aug_MultiplyElementwise = iaa.MultiplyElementwise((0.8, 1.2))
# 随机dropout像素值
self.aug_Dropout = iaa.Dropout(p=(0, 0.2))
# 随机粗dropout,2*2方块像素被dropout
self.aug_CoarseDropout = iaa.CoarseDropout(0.02, size_percent=0.5)
# 50%的图片,p概率反转颜色
self.aug_Invert = iaa.Invert(0.25, per_channel=0.5)
# 对比度归一化
self.aug_ContrastNormalization = iaa.ContrastNormalization((0.5, 1.5))
# 仿射变换
self.aug_Affine = iaa.Affine(rotate=(0, 20),
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
})
# 仿射变换, 局部像素仿射扭曲
self.aug_PiecewiseAffine = iaa.PiecewiseAffine(scale=(0.01, 0.05))
# 单应性变换
self.aug_PerspectiveTransform = iaa.PerspectiveTransform(scale=(0.01,
0.1))
# 弹性变换
self.aug_ElasticTransformation = iaa.ElasticTransformation(alpha=(0,
5.0),
sigma=0.25)
# 简单的加噪,小黑块
self.aug_SimplexNoiseAlpha = iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.0, 0.5)),
iaa.DirectedEdgeDetect(alpha=(0.0, 0.5), direction=(0.0, 1.0)),
]))
# 频域加噪，表现为色彩的块状变换
self.aug_FrequencyNoiseAlpha = iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0)))
import numpy as np
import tensorflow as tf
from imgaug import augmenters as iaa
sometimes_010 = lambda aug: iaa.Sometimes(0.10, aug)
blurs = iaa.Sequential([
sometimes_010(
iaa.OneOf([
iaa.GaussianBlur(sigma=(0, 1.5)),
iaa.AverageBlur(k=(1, 5)),
iaa.MedianBlur(k=(1, 5)),
iaa.MotionBlur(k=(3, 5)),
]))
])
contrasts = iaa.Sequential([
sometimes_010(
iaa.OneOf([
iaa.LogContrast((0.8, 1.2)),
iaa.GammaContrast((0.8, 1.2)),
iaa.LinearContrast((0.8, 1.2)),
iaa.Alpha((0.0, 1.0), iaa.AllChannelsHistogramEqualization()),
iaa.Alpha((0.0, 1.0), iaa.HistogramEqualization()),
iaa.CLAHE(clip_limit=(1, 3)),
iaa.AllChannelsCLAHE(clip_limit=(1, 3)),
]))
])
dropouts = iaa.Sequential([
sometimes_010(
"fc-list :lang=en | sed -r -e 's/^(.+): .*$/\\1/g'",
stdout=subprocess.PIPE,
shell=True).stdout.decode("utf-8").strip().split("\n")
CUSTOM_FONTS = subprocess.run(
"fc-list | sed -r -e 's/^(.+): .*$/\\1/g' | grep custom",
stdout=subprocess.PIPE,
shell=True).stdout.decode("utf-8").strip().split("\n")
FONTNAMES += CUSTOM_FONTS
with open("/usr/share/dict/words") as f:
WORDS = f.read().splitlines()
AUGMENTOR = iaa.Sequential([
iaa.OneOf([iaa.GaussianBlur(
(0, 1.0)), iaa.AverageBlur(k=(1, 2))]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
iaa.Dropout((0.01, 0.03), per_channel=0.1),
iaa.Add((-10, 10), per_channel=0.5),
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(exponent=(-4, 0),
first=iaa.Multiply(
(0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0)))
]),
],
random_order=True)
args = sys.argv
if len(args) != 2:
print("Need option target directory")
sys.exit()
# if args[1] != "train" and args[1] != "val":
# print("Accetable option is 'train' or 'val'")
# sys.exit()
target = args[1]
print("Augment target Directory :", target)
# Augmentation definition
seq1 = iaa.Sequential([
iaa.Sometimes(0.8, iaa.Affine(rotate=(-90, 90))),
# iaa.Sometimes(0.8, iaa.Resize((0.8, 1.2))),
iaa.Sometimes(0.8, iaa.GaussianBlur(sigma=(0.0, 2.0)))
])
seq2 = iaa.Sequential([
iaa.Sometimes(0.8, iaa.LogContrast(gain=(0.8, 1.4))),
iaa.Sometimes(0.4, iaa.AdditiveGaussianNoise(scale=(0, 20)))
])
DATA_DIR = os.path.join(ROOT_DIR, target)
ids = 0
for curDir, dirs, files in os.walk(DATA_DIR):
if not curDir.endswith("augmented") and os.path.isfile(curDir +
"/label.json"):
if not os.path.isdir(curDir + "/augmented"):
os.makedirs(curDir + "/augmented")
annotations = json.load(open(curDir + "/label.json"))
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = BalloonDataset()
dataset_train.load_balloon(args.dataset, "train")
dataset_train.prepare()
# Validation dataset
dataset_val = BalloonDataset()
dataset_val.load_balloon(args.dataset, "val")
dataset_val.prepare()
# Image augmentation
# http://imgaug.readthedocs.io/en/latest/source/augmenters.html
augmentation = iaa.SomeOf(
(1, 5),
[
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Affine(rotate=90), # new to force more horizontal lines
iaa.Affine(
rotate=(-90, 90), mode="edge"
), # mode= "edge" ads straight lines in created empty space
#for this do one of
iaa.OneOf([
iaa.CropAndPad(percent=(-0.3, 0.05),
sample_independently=False,
pad_mode="edge"),
iaa.Crop(percent=(0, 0.05))
]),
iaa.GaussianBlur(sigma=(0.0, 1.0)),
iaa.Multiply((0.5, 1.2)), #changeing brightness
iaa.Grayscale(alpha=(0.0, 0.9))
])
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
print("Training network heads")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=15,
augmentation=augmentation,
layers='heads') # 'heads' or 'all'
print("Training 4+")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=60,
augmentation=augmentation,
layers='4+') # 'heads' or 'all'
print("Train all")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=200,
augmentation=augmentation,
layers='all') # 'heads' or 'all'
print("Train all lower learning rate")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE / 10,
epochs=600,
augmentation=augmentation,
layers='all') # 'heads' or 'all'
#data_transforms = transforms.Compose([
# transforms.Resize((64, 64)),
# transforms.ToTensor(),
# transforms.Normalize(mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225])
#])
augmenter = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Sometimes(0.5, iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode='median')),
iaa.SomeOf((0, 5), [
iaa.Sometimes(
0.5, iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))),
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout(
def load_data(data_dir, data_aug=False):
#files = os.listdir(data_dir)
files = [f for f in os.listdir(data_dir) if not f.startswith('.')]
reduced_height = 240
reduced_width = 135
#X = np.zeros((len(files), 960, 540, 3))
#Y = np.zeros((len(files), 960, 540, 1))
X = np.zeros((len(files), reduced_height, reduced_width, 3))
Y = np.zeros((len(files), reduced_height, reduced_width, 1))
X_aug = np.zeros((len(files), reduced_height, reduced_width, 3))
Y_aug = np.zeros((len(files), reduced_height, reduced_width, 1))
for i, img in enumerate(files):
if img == ".DS_Store" or img == "._.DS_Store":
continue
path = os.path.join(data_dir, img)
# Load image as array (not to be used)
image_array = image.imread(path)
# Load image as PIL object
image_obj = PIL.Image.open(path)
# Crop images
input_img = image_obj.crop((0, 0, 540, 960))
mask_img = image_obj.crop((540, 0, 1080, 960))
# Resize images (here half size)
input_img = input_img.resize((reduced_width, reduced_height),
PIL.Image.ANTIALIAS)
mask_img = mask_img.resize((reduced_width, reduced_height),
PIL.Image.ANTIALIAS)
# Convert to arrays
input_data = np.asarray(input_img)
target_data = np.asarray(mask_img)[:, :, :1]
# Store data
X[i], X_aug[i] = input_data, input_data
Y[i], Y_aug[i] = target_data, target_data
X = np.reshape(X, [len(files), 3, reduced_height, reduced_width])
Y = np.reshape(Y, [len(files), 1, reduced_height, reduced_width])
bg = (Y < 200).astype(int)
hand = (Y >= 200).astype(int)
Y_onehot = np.concatenate((bg, hand), axis=1)
Y = np.argmax(Y_onehot, axis=1)
if data_aug:
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential(
[
iaa.Crop(
px=(10, 40)
), # crop images from each side by 10 to 40px (randomly chosen)
iaa.Fliplr(0.5), # horizontally flip 50% of the images
#sometimes(iaa.ElasticTransformation(alpha=50, sigma=5)),
#iaa.Dropout([0.05, 0.2]),
sometimes(iaa.GaussianBlur(sigma=(0, 5))),
#iaa.Sharpen((0.0, 1.0)),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.Affine(
#scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
#translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
rotate=(-90, 90),
#shear=(-8, 8)
)
],
random_order=True)
X_aug, Y_aug = seq(images=X_aug.astype('uint8'),
segmentation_maps=Y_aug.astype('uint8'))
X_aug = np.reshape(X_aug,
[len(files), 3, reduced_height, reduced_width])
Y_aug = np.reshape(Y_aug,
[len(files), 1, reduced_height, reduced_width])
bg_aug = (Y_aug < 200).astype(int)
hand_aug = (Y_aug >= 200).astype(int)
Y_onehot_aug = np.concatenate((bg_aug, hand_aug), axis=1)
Y_aug = np.argmax(Y_onehot_aug, axis=1)
return X, Y, X_aug, Y_aug
return X, Y
def run_seq():
# Sometimes(0.5, ...) applies the given augmenter in 50% of all cases,
# e.g. Sometimes(0.5, GaussianBlur(0.3)) would blur roughly every second
# image.
# if not isinstance(images,list):
# images=[images]
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image.
seq = iaa.Sequential(
[
#
# Apply the following augmenters to most images.
#
# iaa.Fliplr(0.5), # horizontally flip 50% of all images
# iaa.Flipud(0.2), # vertically flip 20% of all images
# crop some of the images by 0-10% of their height/width
# sometimes(iaa.Crop(percent=(0, 0.1))),
# sometimes(iaa.Crop(percent=(0, 0.05))),
# Apply affine transformations to some of the images
# - scale to 80-120% of image height/width (each axis independently)
# - translate by -20 to +20 relative to height/width (per axis)
# - rotate by -45 to +45 degrees
# - shear by -16 to +16 degrees
# - order: use nearest neighbour or bilinear interpolation (fast)
# - mode: use any available mode to fill newly created pixels
# see API or scikit-image for which modes are available
# - cval: if the mode is constant, then use a random brightness
# for the newly created pixels (e.g. sometimes black,
# sometimes white)
sometimes(
iaa.Affine(
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
# rotate=(-45, 45),
rotate=(-5, 5),
# shear=(-16, 16),
shear=(-5, 5),
order=[0, 1],
# cval=(0, 255),
cval=144, # 填充像素值
# mode=ia.ALL # 默认常数值填充边界
)),
#
# Execute 0 to 5 of the following (less important) augmenters per
# image. Don't execute all of them, as that would often be way too
# strong.
#
iaa.SomeOf(
(0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
sometimes(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(alpha=(0, 0.7),
direction=(0.0, 1.0)),
])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
iaa.Invert(0.05,
per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.LinearContrast((0.5, 2.0), per_channel=0.5),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True)
],
# do all of the above augmentations in random order
random_order=True)
# images_aug = seq(images=images)
return seq
import os
import sys
from imgaug import augmenters as iaa
seq = iaa.Sequential([
iaa.Crop(
px=(0,
16)), # crop images from each side by 0 to 16px (randomly chosen)
iaa.Fliplr(0.5), # horizontally flip 50% of the images
iaa.GaussianBlur(sigma=(0, 3.0)) # blur images with a sigma of 0 to 3.0
])
sys.path.append(os.path.join(os.getcwd(), 'benset'))
from benset_batchloader_ar import *
from benset_dataloader_ar import *
num_frames = 8
dataset_path = "E:\\Bachelorarbeit-SS20\\datasets\\Benset256"
num_action_predictions = 6
benset = Benset(dataset_path, num_action_predictions)
batch_size = 1
benset_seq = BatchLoader(benset, benset.get_train_data_keys(),
benset.get_train_annotations(), batch_size,
num_frames)
x, y = benset_seq.__next__()
def imageAugmentation(dir_in, dir_out):
random.seed(42)
try:
if not os.path.isdir(dir_out):
os.mkdir(dir_out)
except:
pass
list_jpg = glob(dir_in + r"\**\*.jpg", recursive=True)
## https://github.com/albumentations-team/albumentations
list_aug = [
A.CLAHE(),
A.OpticalDistortion(),
# A.GridDistortion(),
# A.HueSaturationValue(),
A.GaussNoise(),
A.MotionBlur(p=.2),
A.RandomBrightnessContrast(p=0.1),
# A.InvertImg(), # Not Accepted
# A.ISONoise(), # Not Accepted
# A.RandomFog(), # Not Accepted
# # A.RandomRain(), # Not Accepted
# # A.RandomSnow() # Not Accepted
]
list_aug_name = [
'CLAHE',
'OpticalDist',
# 'GridDist',
# 'HueSat',
'GaussNoise',
'MotionBlur',
'RandomBright',
# 'InvertImg',
# 'IsoNoise',
# 'RandomFog',
# # 'RandomRain',
# # 'RandomSnow',
]
## https://github.com/aleju/imgaug
list_aa = [
# iaa.MedianBlur(k=(3, 11)), # Not Accepted
# iaa.Dropout((0.05, 0.06), per_channel=0.5), # Not Accepted
# iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # Not Accepted
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, 0.05 * 255),
per_channel=0.5),
iaa.ElasticTransformation(alpha=8.2, sigma=4.0),
# iaa.ElasticTransformation(alpha=15.5, sigma=4.0),
# iaa.ElasticTransformation(alpha=22.8, sigma=4.0),
iaa.PiecewiseAffine(scale=0.015),
iaa.PiecewiseAffine(scale=0.030),
iaa.PiecewiseAffine(scale=0.045),
# iaa.PiecewiseAffine(scale=0.060),
# iaa.PiecewiseAffine(scale=0.075),
iaa.EdgeDetect(alpha=0.3),
# iaa.Sharpen(alpha=(0.0, 1.0)), # Not Accepted
# iaa.DirectedEdgeDetect(alpha=0.5, direction=0), # Not Accepted
iaa.Affine(scale=0.8, mode='edge', cval=64),
iaa.Affine(scale=1.2, mode='edge'),
iaa.Affine(rotate=5, cval=64),
iaa.Affine(rotate=10, cval=64),
iaa.Affine(rotate=-5, cval=64),
iaa.Affine(rotate=-10, cval=64),
iaa.Affine(shear=8, cval=64, mode='edge'),
iaa.Affine(shear=-8, cval=64, mode='edge'),
iaa.Affine(scale=0.8, rotate=3, mode='edge', cval=64),
iaa.Affine(scale=0.8, rotate=-3, mode='edge', cval=64),
iaa.Affine(scale=1.2, rotate=3, mode='edge', cval=64),
iaa.Affine(scale=1.2, rotate=-3, mode='edge', cval=64),
iaa.GaussianBlur(sigma=1.0),
# iaa.MaxPooling(kernel_size=2, keep_size=True),
iaa.Fog(),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(scale=0.8, mode='edge', cval=64),
]),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(scale=1.2, mode='edge'),
]),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(rotate=5, cval=64),
]),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(rotate=10, cval=64),
]),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(rotate=-5, cval=64),
]),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(rotate=-10, cval=64),
]),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(shear=8, cval=64, mode='edge'),
]),
iaa.Sequential([
iaa.GaussianBlur(sigma=1.0),
iaa.Affine(shear=-8, cval=64, mode='edge'),
]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(scale=0.8, mode='edge', cval=64), ]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(scale=1.2, mode='edge'), ]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(rotate=5, cval=64), ]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(rotate=10, cval=64), ]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(rotate=-5, cval=64), ]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(rotate=-10, cval=64), ]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(shear=8, cval=64, mode='edge'), ]),
# iaa.Sequential([iaa.MaxPooling(2, keep_size=True), iaa.Affine(shear=-8, cval=64, mode='edge'), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(scale=0.8, mode='edge', cval=64), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(scale=1.2, mode='edge'), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(rotate=5, cval=64), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(rotate=10, cval=64), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(rotate=-5, cval=64), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(rotate=-10, cval=64), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(shear=8, cval=64, mode='edge'), ]),
# iaa.Sequential([iaa.Fog(), iaa.Affine(shear=-8, cval=64, mode='edge'), ]),
]
list_aa_name = [
# 'MedianBlur',
# 'Dropout',
# 'Emboss',
'AdditiveGaussianNoise',
'ElasticTransformation8',
# 'ElasticTransformation15',
# 'ElasticTransformation22',
'PiecewiseAffine15',
'PiecewiseAffine30',
'PiecewiseAffine45',
# 'PiecewiseAffine60',
# 'PiecewiseAffine75',
'EdgeDetect',
# 'Sharpen',
# 'DirectedEdgeDetect',
'scale8',
'scale12',
'rotate5',
'rotate10',
'rotate_5',
'rotate_10',
'shear8',
'shear_8',
'scale8rotate3',
'scale8rotate_3',
'scale12rotate3',
'scale12rotate_3',
'GaussianBlur',
# 'MaxPooling2',
'Fog',
'GaussianBlurscale8',
'GaussianBlurscale12',
'GaussianBlurrotate5',
'GaussianBlurrotate10',
'GaussianBlurrotate_5',
'GaussianBlurrotate_10',
'GaussianBlurshear8',
'GaussianBlurshear_8',
# 'MaxPooling2scale8',
# 'MaxPooling2scale12',
# 'MaxPooling2rotate5',
# 'MaxPooling2rotate10',
# 'MaxPooling2rotate_5',
# 'MaxPooling2rotate_10',
# 'MaxPooling2shear8',
# 'MaxPooling2shear_8',
# 'Fogscale8',
# 'Fogscale12',
# 'Fogrotate5',
# 'Fogrotate10',
# 'Fogrotate_5',
# 'Fogrotate_10',
# 'Fogshear8',
# 'Fogshear_8',
]
list_torch_tf = [
MyRotationTransform(angles=[-10, -5, 0, 5, 10]),
]
list_torch_tf_name = [
'Rotate',
]
for jpg in list_jpg:
jpg_out = jpg.replace(dir_in, dir_out)
dir_out_jpg = os.path.dirname(jpg_out)
try:
if not os.path.exists(dir_out_jpg):
os.mkdir(dir_out_jpg)
except:
pass
jpg_out_basename = os.path.splitext(jpg_out)[0]
image = imageread(jpg)
imagesave(image, jpg_out_basename + '.jpg') # save the orignal image.
for i in range(len(list_aug)):
augmented_image = list_aug[i](image=image)['image']
imagesave(augmented_image,
jpg_out_basename + '_' + list_aug_name[i] + '.jpg')
for i in range(len(list_aa)):
augmented_image = list_aa[i](image=image)
imagesave(augmented_image,
jpg_out_basename + '_' + list_aa_name[i] + '.jpg')
def det_aug(image, polys_np=None):
"""
随机对图像做以下的增强操作
:param image: cv2 read
:param polys_np:[N, 4, 2]
:return:
"""
aug_sample = random.sample(cfg.TRAIN.AUG_TOOL, 1)[0] #从数组中随机取出一个增强的功能
######################################################################################################
# blur-模糊
aug = None
# 高斯滤波 sigma 为1-10的保留小数点后一位的float的随机值,可根据情况调整
if aug_sample == 'GaussianBlur':
sigma = random.uniform(1, 2)
sigma = round(8, 10)
aug = iaa.GaussianBlur(sigma)
# 平均模糊 k 为1-10的随机 奇 数,范围根据情况调整
if aug_sample == 'AverageBlur':
k = random.randint(8, 10) * 2 + 1
aug = iaa.AverageBlur(k)
# 中值滤波 k 为1-10的随机 奇 数,范围根据情况调整
if aug_sample == 'MedianBlur':
k = random.randint(8, 10) * 2 + 1
aug = iaa.MedianBlur(k)
# 双边滤波 d=1 为 奇 数, sigma_color=(10, 250), sigma_space=(10, 250)
if aug_sample == 'BilateralBlur':
d = random.randint(0, 2) * 2 + 1
sigma_color = random.randint(10, 250)
sigma_space = random.randint(10, 250)
aug = iaa.BilateralBlur(d, sigma_color, sigma_space)
# 运动模糊 k=5 一定大于3 的 奇 数, angle=(0, 360), direction=(-1.0, 1.0)
if aug_sample == 'MotionBlur':
k = random.randint(15, 20) * 2 + 1
angle = random.randint(0, 360)
direction = random.uniform(-1, 1)
direction = round(direction, 1)
aug = iaa.MotionBlur(k, angle, direction)
######################################################################################################
# geometric 几何学
# 弹性变换
if aug_sample == 'ElasticTransformation':
alpha = random.uniform(10, 20)
alpha = round(alpha, 1)
sigma = random.uniform(5, 10)
sigma = round(sigma, 1)
# print(alpha, sigma)
aug = iaa.ElasticTransformation(alpha, sigma)
# 透视
if aug_sample == 'PerspectiveTransform':
scale = random.uniform(0, 0.2)
scale = round(scale, 3)
aug = iaa.PerspectiveTransform(scale)
# 旋转角度
# if aug_sample == 'Affine_rot':
# rotate = random.randint(-20, 20)
# aug = iaa.Affine(rotate=rotate)
# 缩放
# if aug_sample == 'Affine_scale':
# scale = random.uniform(0, 2)
# scale = round(scale, 1)
# aug = iaa.Affine(scale=scale)
######################################################################################################
# flip 镜像
# 水平镜像
# if aug_sample == 'Fliplr':
# aug = iaa.Fliplr(1)
#
# 垂直镜像
# if aug_sample == 'Flipud':
# aug = iaa.Flipud(1)
######################################################################################################
# size 尺寸
# if aug_sample == 'CropAndPad':
# top = random.randint(0, 10)
# right = random.randint(0, 10)
# bottom = random.randint(0, 10)
# left = random.randint(0, 10)
# aug = iaa.CropAndPad(px=(top, right, bottom, left)) # 上 右 下 左 各crop多少像素,然后进行padding
if aug_sample == 'Crop':
top = random.randint(0, 10)
right = random.randint(0, 10)
bottom = random.randint(0, 10)
left = random.randint(0, 10)
aug = iaa.Crop(px=(top, right, bottom, left)) # 上 右 下 左
if aug_sample == 'Pad':
top = random.randint(0, 10)
right = random.randint(0, 10)
bottom = random.randint(0, 10)
left = random.randint(0, 10)
aug = iaa.Pad(px=(top, right, bottom, left)) # 上 右 下 左
# if aug_sample == 'PadToFixedSize':
# height = image.shape[0] + 32
# width = image.shape[1] + 100
# aug = iaa.PadToFixedSize(width=width, height=height)z
# if aug_sample == 'CropToFixedSize':
# height = image.shape[0] - 32
# width = image.shape[1] - 100
# aug = iaa.CropToFixedSize(width=width, height=height)
if polys_np is not None:
if aug is not None:
# print(aug_sample)
h, w, _ = image.shape
boxes_info_list = []
for box in polys_np:
boxes_info_list.append(Polygon(box))
psoi = ia.PolygonsOnImage(boxes_info_list,
shape=image.shape) # 生成单个图像上所有多边形的对象
image, psoi_aug = aug(image=image, polygons=psoi)
pts_list = []
for each_poly in psoi_aug.polygons:
pts_list.append(np.array(each_poly.exterior).reshape((4, 2)))
return image, np.array(pts_list, np.float32).reshape((-1, 4, 2))
else:
return image, polys_np
else:
image = aug(image=image)
return image
def get_augmenter():
seq = iaa.Sequential([
iaa.GaussianBlur(sigma=(config.blur_min, config.blur_max)),
iaa.SaltAndPepper((config.s_p_min, config.s_p_max))
])
return seq
def _get_aug():
import imgaug as ia
from imgaug import augmenters as iaa
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
return iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop images by -5% to 10% of their height/width
sometimes(iaa.CropAndPad(
percent=(-0.05, 0.1),
pad_mode=ia.ALL,
pad_cval=(0, 255)
)),
sometimes(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf((0, 5),
[
sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0))
)
]),
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True
)
],
random_order=True
)
def main():
step = 1
experiment_options = Options()
global args
# config parameters
args = experiment_options.args
experiment_name = args.experiment_name
dataset_name = args.dataset_name
number_of_workers = args.num_workers
resume = args.resume
hyperparameters = experiment_options.GetHyperparameters(step, dataset_name)
# training parameters
batchSize = hyperparameters.batchSize
weight_decay = hyperparameters.weight_decay
lr = hyperparameters.lr
number_of_clusters = hyperparameters.number_of_clusters
number_of_clustering_rounds = hyperparameters.number_of_clustering_rounds
nms_thres_superpoint = hyperparameters.nms_thres_superpoint
confidence_thres_superpoint = hyperparameters.confidence_thres_superpoint
use_box = hyperparameters.use_box
remove_superpoint_outliers_percentage = hyperparameters.remove_superpoint_outliers_percentage
training_iterations_before_first_clustering = hyperparameters.training_iterations_before_first_clustering
confidence_thres_FAN = hyperparameters.confidence_thres_FAN
UseScales = hyperparameters.UseScales
RemoveBackgroundClusters = hyperparameters.RemoveBackgroundClusters
#load paths
with open('paths/main.yaml') as file:
paths = yaml.load(file, Loader=yaml.FullLoader)
CheckPaths(paths, dataset_name)
log_path = paths['log_path']
path_to_superpoint_checkpoint = paths['path_to_superpoint_checkpoint']
#This funcion will create the directories /Logs and a /CheckPoints at log_path
Utils.initialize_log_dirs(experiment_name, log_path)
LogText(
f"Experiment Name {experiment_name}\n"
f"Database {dataset_name}\n"
"Training Parameters: \n"
f"Batch size {batchSize} \n"
f"Learning rate {lr} \n"
f"Weight Decay {weight_decay} \n"
f"Training iterations before first clustering {training_iterations_before_first_clustering} \n"
f"Number of clustering rounds {number_of_clustering_rounds} \n"
f"FAN detection threshold {confidence_thres_FAN} \n"
f"Number of Clusters {number_of_clusters} \n"
f"Outlier removal {remove_superpoint_outliers_percentage} \n",
experiment_name, log_path)
LogText("Training of First step begins", experiment_name, log_path)
#augmentations for first step
augmentations = iaa.Sequential([
iaa.Sometimes(0.3, iaa.GaussianBlur(sigma=(0, 0.5))),
iaa.ContrastNormalization((0.85, 1.3)),
iaa.Sometimes(
0.5,
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, 0.05 * 255),
per_channel=0.5)),
iaa.Multiply((0.9, 1.1), per_channel=0.2),
iaa.Sometimes(
0.3,
iaa.MultiplyHueAndSaturation((0.5, 1.5), per_channel=True),
),
iaa.Affine(
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.05, 0.05),
"y": (-0.05, 0.05)
},
rotate=(-40, 40),
)
])
#selection of the dataloading function
superpoint_dataloading_function = Database.get_image_superpoint
if (UseScales):
superpoint_dataloading_function = Database.get_image_superpoint_multiple_scales
superpoint = SuperPoint(
number_of_clusters,
confidence_thres_superpoint,
nms_thres_superpoint,
path_to_superpoint_checkpoint,
experiment_name,
log_path,
remove_superpoint_outliers_percentage,
use_box,
UseScales,
RemoveBackgroundClusters,
)
superpoint_dataset = Database(
dataset_name,
number_of_clusters,
function_for_dataloading=superpoint_dataloading_function,
augmentations=augmentations,
use_box=use_box)
dataloader = DataLoader(superpoint_dataset,
batch_size=batchSize,
shuffle=False,
num_workers=number_of_workers,
drop_last=True)
criterion = nn.MSELoss().cuda()
FAN = FAN_Model(number_of_clusters, criterion, experiment_name,
confidence_thres_FAN, log_path, step)
FAN.init_firststep(lr, weight_decay, number_of_clusters,
training_iterations_before_first_clustering)
if (resume):
path_to_checkpoint, path_to_keypoints = Utils.GetPathsResumeFirstStep(
experiment_name, log_path)
if (path_to_checkpoint is not None):
FAN.load_trained_fiststep_model(path_to_checkpoint)
keypoints = Utils.load_keypoints(path_to_keypoints)
else:
#get initial pseudo-groundtruth by applying superpoint on the training data
keypoints = superpoint.CreateInitialPseudoGroundtruth(dataloader)
dataset = Database(
dataset_name,
FAN.number_of_clusters,
image_keypoints=keypoints,
function_for_dataloading=Database.get_FAN_firstStep_train,
augmentations=augmentations)
dataloader = DataLoader(dataset,
batch_size=batchSize,
shuffle=True,
num_workers=number_of_workers,
drop_last=True)
database_clustering = Database(
dataset_name,
FAN.number_of_clusters,
function_for_dataloading=Database.get_FAN_inference)
dataloader_clustering = DataLoader(database_clustering,
batch_size=batchSize,
shuffle=False,
num_workers=number_of_workers,
drop_last=True)
for i in range(number_of_clustering_rounds):
FAN.Train_step1(dataloader)
keypoints = FAN.Update_pseudoLabels(dataloader_clustering, keypoints)
dataset = Database(
dataset_name,
FAN.number_of_clusters,
image_keypoints=keypoints,
function_for_dataloading=Database.get_FAN_firstStep_train,
augmentations=augmentations)
dataloader = DataLoader(dataset,
batch_size=batchSize,
shuffle=True,
num_workers=number_of_workers,
drop_last=True)
def get_aug_methods():
# options: ["constant", "edge", "symmetric", "reflect", "wrap"]
mode = "edge"
seq = np.array([
# rotate + shear + translate + scale
iaa.Affine(rotate=(-20, 20), shear=(-20, 20), translate_percent=(-.2, .2), scale=(0.9, 1.0), order=[0, 1, 2, 3], mode=mode),
# shear + rotate + translate + scale
iaa.Affine(shear=(-20, 20), rotate=(-20, 20), translate_percent=(-.2, .2), scale=(0.9, 1.0), order=[0, 1, 2, 3], mode=mode),
# scale + shear + rotate + translate + scale
iaa.Affine(scale=(0.85, 1.0), shear=(-20, 20), rotate=(-20, 20), translate_percent=(-.2, .2), order=[0, 1, 2, 3], mode=mode),
# rotate + translate + shear + scale
iaa.Affine(rotate=(-20, 20), translate_percent=(-.2, .2), shear=(-20, 20), scale=(0.9, 1.0), order=[0, 1, 2, 3], mode=mode),
# rotate + translate + scale
iaa.Affine(rotate=(-20, 20), translate_percent=(-.2, .2), scale=(0.9, 1.0), order=[0, 1, 2], mode=mode),
# scale + rotate + translate
iaa.Affine(scale=(0.9, 1.0), rotate=(-20, 20), translate_percent=(-.2, .2), order=[0, 1, 2], mode=mode),
# crop + translate + shear
iaa.Sequential([
iaa.Crop(percent=(0, 0.1)),
iaa.Affine(translate_percent=(-.1, .1), shear=(-15, 15), order=[0,1], mode=mode)
]),
# crop + rotate + translate + shear
iaa.Sequential([
iaa.Crop(percent=(0, 0.1)),
iaa.Affine(rotate=(-10,10), translate_percent=(-.1, .1), shear=(-15, 15), order=[0, 1, 2], mode=mode)
]),
# crop + rotate + translate + shear + flipr
iaa.Sequential([
iaa.Crop(percent=(0, 0.1)),
iaa.Affine(rotate=(-10, 10), translate_percent=(-.1, .1), shear=(-15, 15), order=[0, 1, 2], mode=mode),
iaa.Fliplr(0.9)
]),
# gaussian noise
iaa.AdditiveGaussianNoise(scale=(0, 0.08 * 255)),
# sharpen
iaa.Sharpen(alpha=(0.5, 1.0), lightness=(0.7, 2.)),
# crop
iaa.Crop(percent=(0, 0.15)),
# gaussian + flipr
iaa.Sequential([
iaa.AdditiveGaussianNoise(scale=(0, 0.08 * 255)),
iaa.Fliplr(0.9)
]),
# sharpen + flipr
iaa.Sequential([
iaa.Sharpen(alpha=(0.5, 1.0), lightness=(0.7, 2.)),
iaa.Fliplr(0.9)
]),
# crop + flipr
iaa.Sequential([
iaa.Crop(percent=(0, 0.15)),
iaa.Fliplr(0.9)
]),
# gaussian blur
iaa.GaussianBlur(sigma=(1.,1.5)),
# add
iaa.Add((-20, 40)),
# contrast normalization
iaa.ContrastNormalization((1., 1.5)),
# piecewise affine 1
iaa.PiecewiseAffine(scale=(0.01, 0.05)),
# piecewise affine 2
iaa.PiecewiseAffine(scale=(0.01, 0.05)),
# piecewise affine 3
iaa.PiecewiseAffine(scale=(0.01, 0.05)),
# rotate + translate + scale + piecewise affine
iaa.Sequential([
iaa.Affine(rotate=(-20, 20), translate_percent=(-.2, .2), scale=(0.9, 1.0), order=[0, 1, 2], mode=mode),
iaa.PiecewiseAffine(scale=(0.01, 0.05))
])
])
return seq
def __init__(self,
ann_file,
root,
remove_images_without_annotations,
mask_type='polygon',
transforms=None):
# "mask_type" = "polygon" or "image"
self.mask_type = mask_type
self.img_key_list = list()
self.img_dict = dict()
self.ann_info = dict()
cls = RSNADataset.CLASSES
self.class_to_ind = dict(zip(cls, range(len(cls))))
for dirName, subdirList, fileList in os.walk(root):
for filename in fileList:
filename, ext = os.path.splitext(filename)
if ext.lower() in [".png", ".jpg", ".jpeg"]:
self.img_dict[filename] = os.path.join(
dirName, filename + ext)
self.ann_info[filename] = list()
# csv 용도 이며, mask 이미지인 경우 다르게 작업
with open(ann_file, 'r') as ann_f:
ann_cvf = csv.reader(ann_f)
# patientId,x,y,width,height,Target
for i, line in enumerate(ann_cvf):
if i == 0:
continue
filename, x, y, w, h, target = line
target = int(target)
if remove_images_without_annotations:
if target == 0:
continue
x1 = int(x)
y1 = int(y)
w = int(w)
h = int(h)
x2 = x1 + w
y2 = y1 + h
self.img_key_list.append(filename)
else:
self.img_key_list.append(filename)
x1 = 0
y1 = 0
x2 = 0
y2 = 0
try:
self.ann_info[filename].append([x1, y1, x2, y2, target])
except KeyError:
continue
self.augmentation = iaa.Sequential([
iaa.OneOf([ ## geometric transform
iaa.Affine(
scale={
"x": (0.98, 1.02),
"y": (0.98, 1.02)
},
translate_percent={
"x": (-0.02, 0.02),
"y": (-0.04, 0.04)
},
rotate=(-2, 2),
shear=(-1, 1),
),
iaa.PiecewiseAffine(scale=(0.001, 0.025)),
]),
iaa.OneOf([ ## brightness or contrast
iaa.Multiply((0.9, 1.1)),
iaa.ContrastNormalization((0.9, 1.1)),
]),
iaa.OneOf([ ## blur or sharpen
iaa.GaussianBlur(sigma=(0.0, 0.1)),
iaa.Sharpen(alpha=(0.0, 0.1)),
]),
])
# 중복 방지 RSNA csv 파일 참조
self.img_key_list = list(set(self.img_key_list))
self.transforms = transforms
''' Max Data Aug'''
seq_1 = iaa.Sequential([
# these are essential augumentations
iaa.Fliplr(0.5),
iaa.Flipud(0.3),
iaa.Crop(px=(0, 16)),
# these are less important means only a few of the following will be applied
iaa.SomeOf((0, 6), [
iaa.GammaContrast(1.5),
iaa.Affine(translate_percent={"x": (-0.3, 0.3), "y": (-0.3, 0.3)}, scale=(0.5, 0.9)),
iaa.Affine(rotate=(-60, 60)),
iaa.Multiply((0.5, 1.5)),
iaa.AdditiveGaussianNoise(scale=(10, 60)),
iaa.GaussianBlur(sigma=(0, 0.5))
],random_order=True),
], random_order=True)
''' Optmized Data Aug'''
seq_2 = iaa.Sequential([
iaa.Fliplr(0.6),
iaa.Flipud(0.4),
iaa.Affine(rotate=(-25, 25)),
iaa.Crop((200, 400), keep_size=True)
# some of the follwing will be applied
# iaa.SomeOf((0, 3), [
# iaa.Crop(px=(0, 70)),
def normal_train():
pneumothorax_anns, pneumothorax_fps_train, train_names, test_names, image_annotations = load_data(
DATA_DIR, first=100)
image_fps_train, image_fps_val = train_test_split(pneumothorax_fps_train,
test_size=0.1,
random_state=42)
if debug:
print('DEBUG subsampling from:', len(image_fps_train),
len(image_fps_val), len(test_names))
image_fps_train = image_fps_train[:150]
image_fps_val = test_names[:150]
# test_image_fps = test_names[:150]
print(len(image_fps_train), len(image_fps_val), len(test_names))
# Original image size: 1024 x 1024
ORIG_SIZE = 1024
## Create and prepare the training dataset
# %%
# prepare the training dataset
dataset_train = DetectorDataset(image_fps_train, image_annotations,
ORIG_SIZE, ORIG_SIZE)
dataset_train.prepare()
# %%
# prepare the validation dataset
dataset_val = DetectorDataset(image_fps_val, image_annotations, ORIG_SIZE,
ORIG_SIZE)
dataset_val.prepare()
## Display a random image with bounding boxes
# %%
# Load and display random sample and their bounding boxes
class_ids = [0]
i = 0
while class_ids[0] == 0: ## look for a mask
image_id = random.choice(dataset_val.image_ids)
image_fp = dataset_val.image_reference(image_id)
image = dataset_val.load_image(image_id)
mask, class_ids = dataset_val.load_mask(image_id)
print("Loaded %d %s %s: " % (i, class_ids, image_id),
end="\r",
flush=True)
i += 1
print(image.shape)
plt.figure(figsize=(10, 10))
plt.subplot(1, 2, 1)
plt.imshow(image)
plt.axis('off')
plt.subplot(1, 2, 2)
masked = np.zeros(image.shape[:2])
for i in range(mask.shape[2]):
masked += image[:, :, 0] * mask[:, :, i]
plt.imshow(masked, cmap='gray')
plt.axis('off')
print(image_fp)
print(class_ids)
## Image Augmentation
# Image augmentation (light but constant)
augmentation = iaa.Sequential([
iaa.OneOf([ ## geometric transform
iaa.Affine(
scale={
"x": (0.98, 1.02),
"y": (0.98, 1.04)
},
translate_percent={
"x": (-0.02, 0.02),
"y": (-0.04, 0.04)
},
rotate=(-2, 2),
shear=(-1, 1),
),
iaa.PiecewiseAffine(scale=(0.001, 0.025)),
]),
iaa.OneOf([ ## brightness or contrast
iaa.Multiply((0.9, 1.1)),
iaa.ContrastNormalization((0.9, 1.1)),
]),
iaa.OneOf([ ## blur or sharpen
iaa.GaussianBlur(sigma=(0.0, 0.1)),
iaa.Sharpen(alpha=(0.0, 0.1)),
]),
])
# test on the same image as above
imggrid = augmentation.draw_grid(image[:, :, 0], cols=5, rows=2)
plt.figure(figsize=(30, 12))
_ = plt.imshow(imggrid[:, :, 0], cmap='gray')
# get pixel statistics
images = []
for image_id in dataset_val.image_ids:
image = dataset_val.load_image(image_id)
images.append(image)
images = np.array(images)
config = DetectorConfig()
# config.display()
config.MEAN_PIXEL = images.mean(axis=(0, 1, 2)).tolist()
VAR_PIXEL = images.var()
print(config.MEAN_PIXEL, VAR_PIXEL)
model = modellib.MaskRCNN(mode='training',
config=config,
model_dir=ROOT_DIR)
# Train the Mask-RCNN Model
LEARNING_RATE = 0.0006
# %%
## train heads with higher lr to speedup the learning
model.train(dataset_train,
dataset_val,
learning_rate=LEARNING_RATE * 10,
epochs=1,
layers='heads',
augmentation=None) ## no need to augment yet
history = model.keras_model.history.history
# %%
model.train(dataset_train,
dataset_val,
learning_rate=LEARNING_RATE,
epochs=3 if debug else 11,
layers='all',
augmentation=augmentation)
new_history = model.keras_model.history.history
for k in new_history:
history[k] = history[k] + new_history[k]
# %%
model.train(dataset_train,
dataset_val,
learning_rate=LEARNING_RATE / 2,
epochs=4 if debug else 18,
layers='all',
augmentation=augmentation)
new_history = model.keras_model.history.history
for k in new_history:
history[k] = history[k] + new_history[k]
# %%
epochs = range(1, len(history['loss']) + 1)
pd.DataFrame(history, index=epochs)
# %%
plt.figure(figsize=(21, 11))
plt.subplot(231)
plt.plot(epochs, history["loss"], label="Train loss")
plt.plot(epochs, history["val_loss"], label="Valid loss")
plt.legend()
plt.subplot(232)
plt.plot(epochs, history["rpn_class_loss"], label="Train RPN class ce")
plt.plot(epochs, history["val_rpn_class_loss"], label="Valid RPN class ce")
plt.legend()
plt.subplot(233)
plt.plot(epochs, history["rpn_bbox_loss"], label="Train RPN box loss")
plt.plot(epochs, history["val_rpn_bbox_loss"], label="Valid RPN box loss")
plt.legend()
plt.subplot(234)
plt.plot(epochs, history["mrcnn_class_loss"], label="Train MRCNN class ce")
plt.plot(epochs,
history["val_mrcnn_class_loss"],
label="Valid MRCNN class ce")
plt.legend()
plt.subplot(235)
plt.plot(epochs, history["mrcnn_bbox_loss"], label="Train MRCNN box loss")
plt.plot(epochs,
history["val_mrcnn_bbox_loss"],
label="Valid MRCNN box loss")
plt.legend()
plt.subplot(236)
plt.plot(epochs, history["mrcnn_mask_loss"], label="Train Mask loss")
plt.plot(epochs, history["val_mrcnn_mask_loss"], label="Valid Mask loss")
plt.legend()
plt.show()
# %%
best_epoch = np.argmin(history["val_loss"])
score = history["val_loss"][best_epoch]
print(f'Best Epoch:{best_epoch + 1} val_loss:{score}')
# %%
# select trained model
dir_names = next(os.walk(model.model_dir))[1]
key = config.NAME.lower()
dir_names = filter(lambda f: f.startswith(key), dir_names)
dir_names = sorted(dir_names)
if not dir_names:
import errno
raise FileNotFoundError(
errno.ENOENT,
"Could not find model directory under {}".format(model.model_dir))
fps = []
# Pick last directory
for d in dir_names:
dir_name = os.path.join(model.model_dir, d)
# Find checkpoints
checkpoints = next(os.walk(dir_name))[2]
checkpoints = filter(lambda f: f.startswith("mask_rcnn"), checkpoints)
checkpoints = sorted(checkpoints)
if not checkpoints:
raise Exception(f'No weight files in {dir_name}')
if best_epoch < len(checkpoints):
checkpoint = checkpoints[best_epoch]
else:
checkpoint = checkpoints[-1]
fps.append(os.path.join(dir_name, checkpoint))
model_path = sorted(fps)[-1]
print('Found model {}'.format(model_path))
inference_config = InferenceConfig()
# Recreate the model in inference mode
model = modellib.MaskRCNN(mode='inference',
config=inference_config,
model_dir=ROOT_DIR)
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
# set color for class
def get_colors_for_class_ids(class_ids):
colors = []
for class_id in class_ids:
if class_id == 1:
colors.append((.941, .204, .204))
return colors
# Show few example of ground truth vs. predictions on the validation dataset
dataset = dataset_val
pneumothorax_ids_val = [fp.split('/')[-1][:-4] for fp in image_fps_val]
pneumothorax_ids_val = [
i for i, id in enumerate(pneumothorax_ids_val)
if id in pneumothorax_anns
]
fig = plt.figure(figsize=(10, 40))
for i in range(8):
image_id = random.choice(pneumothorax_ids_val)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
# print(original_image.shape)
plt.subplot(8, 2, 2 * i + 1)
visualize.display_instances(
original_image,
gt_bbox,
gt_mask,
gt_class_id,
dataset.class_names,
colors=get_colors_for_class_ids(gt_class_id),
ax=fig.axes[-1])
plt.subplot(8, 2, 2 * i + 2)
results = model.detect([original_image]) # , verbose=1)
r = results[0]
visualize.display_instances(original_image,
r['rois'],
r['masks'],
r['class_ids'],
dataset.class_names,
r['scores'],
colors=get_colors_for_class_ids(
r['class_ids']),
ax=fig.axes[-1])
## Basic Model Weigths Analysis
# %%
# Show stats of all trainable weights
visualize.display_weight_stats(model)
### Click to expand output
# %%
# from https://github.com/matterport/Mask_RCNN/blob/master/samples/coco/inspect_weights.ipynb
# Pick layer types to display
LAYER_TYPES = ['Conv2D', 'Dense', 'Conv2DTranspose']
# Get layers
layers = model.get_trainable_layers()
layers = list(filter(lambda l: l.__class__.__name__ in LAYER_TYPES,
layers))
# Display Histograms
fig, ax = plt.subplots(len(layers),
2,
figsize=(10, 3 * len(layers)),
gridspec_kw={"hspace": 1})
for l, layer in enumerate(layers):
weights = layer.get_weights()
for w, weight in enumerate(weights):
tensor = layer.weights[w]
ax[l, w].set_title(f'Layer:{l}.{w} {tensor.name}')
_ = ax[l, w].hist(weight[w].flatten(), 50)
sub = pd.read_csv(
'/kaggle/input/siim-acr-pneumothorax-segmentation/sample_submission.csv'
)
# based on https://www.kaggle.com/raddar/better-sample-submission Will this work in stage 2?
positives = sub.groupby('ImageId').ImageId.count().reset_index(
name='N').set_index('ImageId')
positives = positives.loc[
positives.N >
1] # find image id's with more than 1 row -> has pneumothorax mask!
positives.head()
# %%
# Make predictions on test images, write out submission file
# %%
submission_fp = os.path.join(ROOT_DIR, 'submission.csv')
predict(test_names, filepath=submission_fp)
print(submission_fp)
# %%
sub = pd.read_csv(submission_fp)
print((sub.EncodedPixels != '-1').sum(), sub.ImageId.size,
sub.ImageId.nunique())
print(sub.EncodedPixels.nunique(),
(sub.EncodedPixels != '-1').sum() / sub.ImageId.nunique())
print('Unique samples:\n', sub.EncodedPixels.drop_duplicates()[:6])
sub.head(10)
# %%
# show a few test image detection example
for i in range(8):
visualize_test()
# %%
os.chdir(ROOT_DIR)
def GenerateRandomImgaugAugmentation(
pNbAugmentations=5, # number of augmentations
pEnableResizing=True, # enable scaling
pScaleFactor=0.5, # maximum scale factor
pEnableCropping=True, # enable cropping
pCropFactor=0.25, # maximum crop out size (minimum new size is 1.0-pCropFactor)
pEnableFlipping1=True, # enable x flipping
pEnableFlipping2=True, # enable y flipping
pEnableRotation90=True, # enable rotation
pEnableRotation=True, # enable rotation
pMaxRotationDegree=15, # maximum shear degree
pEnableShearX=True, # enable x shear
pEnableShearY=True, # enable y shear
pMaxShearDegree=15, # maximum shear degree
pEnableDropOut=True, # enable pixel dropout
pMaxDropoutPercentage=.1, # maximum dropout percentage
pEnableBlur=True, # enable gaussian blur
pBlurSigma=.25, # maximum sigma for gaussian blur
pEnableSharpness=True, # enable sharpness
pSharpnessFactor=.1, # maximum additional sharpness
pEnableEmboss=True, # enable emboss
pEmbossFactor=.1, # maximum emboss
pEnableBrightness=True, # enable brightness
pBrightnessFactor=.1, # maximum +- brightness
pEnableRandomNoise=True, # enable random noise
pMaxRandomNoise=.1, # maximum random noise strength
pEnableInvert=False, # enables color invert
pEnableContrast=True, # enable contrast change
pContrastFactor=.1, # maximum +- contrast
):
augmentationMap = []
augmentationMapOutput = []
if pEnableResizing:
if random.Random().randint(0, 1) == 1:
randomResizeX = 1 - random.Random().random() * pScaleFactor
else:
randomResizeX = 1 + random.Random().random() * pScaleFactor
if random.Random().randint(0, 1) == 1:
randomResizeY = 1 - random.Random().random() * pScaleFactor
else:
randomResizeY = 1 + random.Random().random() * pScaleFactor
aug = iaa.Resize({"height": randomResizeY, "width": randomResizeX})
augmentationMap.append(aug)
if pEnableCropping:
randomCrop2 = random.Random().random() * pCropFactor
randomCrop4 = random.Random().random() * pCropFactor
randomCrop1 = random.Random().random() * pCropFactor
randomCrop3 = random.Random().random() * pCropFactor
aug = iaa.Crop(percent=(randomCrop1, randomCrop2, randomCrop3,
randomCrop4))
augmentationMap.append(aug)
if pEnableFlipping1:
aug = iaa.Fliplr()
augmentationMap.append(aug)
if pEnableFlipping2:
aug = iaa.Flipud()
augmentationMap.append(aug)
if pEnableRotation90:
randomNumber = random.Random().randint(1, 3)
aug = iaa.Rot90(randomNumber)
augmentationMap.append(aug)
if pEnableRotation:
if random.Random().randint(0, 1) == 1:
randomRotation = random.Random().random() * pMaxRotationDegree
else:
randomRotation = -random.Random().random() * pMaxRotationDegree
aug = iaa.Rotate(randomRotation)
augmentationMap.append(aug)
if pEnableShearX:
if random.Random().randint(0, 1) == 1:
randomShearingX = random.Random().random() * pMaxShearDegree
else:
randomShearingX = -random.Random().random() * pMaxShearDegree
aug = iaa.ShearX(randomShearingX)
augmentationMap.append(aug)
if pEnableShearY:
if random.Random().randint(0, 1) == 1:
randomShearingY = random.Random().random() * pMaxShearDegree
else:
randomShearingY = -random.Random().random() * pMaxShearDegree
aug = iaa.ShearY(randomShearingY)
augmentationMap.append(aug)
if pEnableDropOut:
randomDropOut = random.Random().random() * pMaxDropoutPercentage
aug = iaa.Dropout(p=randomDropOut, per_channel=False)
augmentationMap.append(aug)
if pEnableBlur:
randomBlur = random.Random().random() * pBlurSigma
aug = iaa.GaussianBlur(randomBlur)
augmentationMap.append(aug)
if pEnableSharpness:
randomSharpness = random.Random().random() * pSharpnessFactor
aug = iaa.Sharpen(randomSharpness)
augmentationMap.append(aug)
if pEnableEmboss:
randomEmboss = random.Random().random() * pEmbossFactor
aug = iaa.Emboss(randomEmboss)
augmentationMap.append(aug)
if pEnableBrightness:
if random.Random().randint(0, 1) == 1:
randomBrightness = 1 - random.Random().random() * pBrightnessFactor
else:
randomBrightness = 1 + random.Random().random() * pBrightnessFactor
aug = iaa.Add(randomBrightness)
augmentationMap.append(aug)
if pEnableRandomNoise:
if random.Random().randint(0, 1) == 1:
randomNoise = 1 - random.Random().random() * pMaxRandomNoise
else:
randomNoise = 1 + random.Random().random() * pMaxRandomNoise
aug = iaa.MultiplyElementwise(randomNoise, per_channel=True)
augmentationMap.append(aug)
if pEnableInvert:
aug = iaa.Invert(1)
augmentationMap.append(aug)
if pEnableContrast:
if random.Random().randint(0, 1) == 1:
randomContrast = 1 - random.Random().random() * pContrastFactor
else:
randomContrast = 1 + random.Random().random() * pContrastFactor
aug = iaa.contrast.LinearContrast(randomContrast)
augmentationMap.append(aug)
widthFactor = 1
heightFactor = 1
arr = numpy.arange(0, len(augmentationMap))
numpy.random.shuffle(arr)
switchWidthHeight = False
for i in range(pNbAugmentations):
augmentationMapOutput.append(augmentationMap[arr[i]])
if arr[i] == 0:
widthFactor *= randomResizeX
heightFactor *= randomResizeY
if arr[i] == 1:
widthFactor *= (1.0 - (randomCrop2 + randomCrop4))
heightFactor *= (1.0 - (randomCrop1 + randomCrop3))
if arr[i] == 4:
if randomNumber == 1 or randomNumber == 3:
switchWidhtHeight = True
return iaa.Sequential(
augmentationMapOutput), widthFactor, heightFactor, switchWidthHeight
def __init__(self):
configMain.__init__(self)
st = lambda aug: iaa.Sometimes(0.4, aug)
oc = lambda aug: iaa.Sometimes(0.3, aug)
rl = lambda aug: iaa.Sometimes(0.09, aug)
self.augment = iaa.Sequential(
[
rl(iaa.GaussianBlur(
(0, 1.5))), # blur images with a sigma between 0 and 1.5
rl(
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05),
per_channel=0.5)), # add gaussian noise to images
oc(iaa.Dropout((0.0, 0.10), per_channel=0.5)
), # randomly remove up to X% of the pixels
oc(
iaa.CoarseDropout(
(0.0, 0.10), size_percent=(0.08, 0.2), per_channel=0.5)
), # randomly remove up to X% of the pixels
oc(
iaa.Add((-40, 40), per_channel=0.5)
), # change brightness of images (by -X to Y of original value)
st(iaa.Multiply((0.10, 2.5), per_channel=0.2)
), # change brightness of images (X-Y% of original value)
rl(iaa.ContrastNormalization(
(0.5, 1.5),
per_channel=0.5)), # improve or worsen the contrast
rl(iaa.Grayscale((0.0, 1))), # put grayscale
],
random_order=True # do all of the above in random order
)
self.augment_labels = True
self.augment_amount = 1 #3=max, 2=mid, 1=min
self.labels_to_augment = {
"road": True,
"buildings": True,
"grass": True,
"sky_n_zebra": True
}
# there are files with data, 200 images each, and here we select which ones to use
#5self.dataset_name = 'Carla'
#with open(os.path.join(self.save_data_stats, 'path'),'r') as f:
# path = f.read().strip()
path = '../VirtualElektraData2'
train_path = os.path.join(path, 'SeqTrain')
val_path = os.path.join(path, 'SeqVal')
print train_path, val_path
self.train_db_path = [
os.path.join(train_path, f)
for f in glob.glob1(train_path, "data_*.h5")
]
self.val_db_path = [
os.path.join(val_path, f)
for f in glob.glob1(val_path, "data_*.h5")
]
# When using data with noise, remove the recording during the first half of the noise impulse
# TODO Felipe: change to noise percentage.
self.remove_noise = False
# Speed Divide Factor
#TODO: FOr now is hardcooded, but eventually we should be able to calculate this from data at the loading time.
self.speed_factor = 1.0 # In KM/H FOR GTA it should be maximun 30.0
# The division is made by three diferent data kinds
# in every mini-batch there will be equal number of samples with labels from each group
# e.g. for [[0,1],[2]] there will be 50% samples with labels 0 and 1, and 50% samples with label 2
self.labels_per_division = [[2], [2], [2]]
self.dataset_names = ['targets']
self.queue_capacity = 20 * self.batch_size
# TODO NOT IMPLEMENTED Felipe: True/False switches to turn data balancing on or off
self.balances_val = True
self.balances_train = True
self.augment_and_saturate_factor = True
originals = path.joinpath("Originals")
masks = path.joinpath("Masks")
result_originals = path.joinpath("AugmentedOriginals")
result_masks = path.joinpath("AugmentedMasks")
for i in range(IMAGE_COUNT):
istr = str(i % 900) + ".jpg"
original = imageio.imread(originals.joinpath(istr))
mask = imageio.imread(masks.joinpath(istr))
mask = SegmentationMapsOnImage(mask, shape=mask.shape)
seq = iaa.SomeOf((0, None), random_order=True)
seq.add(iaa.Add((-40, 40), per_channel=0.5))
seq.add(iaa.GaussianBlur(sigma=(0, 2)))
seq.add(iaa.SigmoidContrast(gain=(5, 20), cutoff=(0.3, 0.75), per_channel=True))
seq.add(iaa.HorizontalFlip())
seq.add(iaa.VerticalFlip())
seq.add(iaa.TranslateX(percent=(-0.7, 0.7), cval=33))
seq.add(iaa.TranslateY(percent=(-0.7, 0.7), cval=33))
seq.add(iaa.Rotate(random.randrange(-60, 60), cval=33))
seq.add(iaa.ScaleX((0.5, 1.5), cval=33))
seq.add(iaa.ScaleY((0.5, 1.5), cval=33))
seq.add(iaa.imgcorruptlike.DefocusBlur(severity=1))
aug = iaa.CropAndPad(percent=([-0.3, 0.3], [-0.3, 0.3], [-0.3, 0.3], [-0.3, 0.3]))
results_o, results_m = seq(image=original, segmentation_maps=mask)
istr = str(i) + ".jpg"
imageio.imsave(result_originals.joinpath(istr), results_o)
def __init__(self, image_root, txt_path, is_train, transform=None):
super(Single_Dataset, self).__init__()
self.image_root = image_root
self.txt_path = txt_path
self.is_train = is_train
self.transform = transform
self.image_list = []
self.label_list = []
lines = open(self.txt_path, 'r').readlines()
lines = [line.strip() for line in lines]
for line in lines:
self.image_list.append(line.split(' ')[0])
self.label_list.append(int(line.split(' ')[1]))
self.seq = iaa.SomeOf(
(3, 11),
{
# self.seq = iaa.SomeOf((0, 5), {
# iaa.Fliplr(0.5),
iaa.Flipud(0.5),
# iaa.Crop(percent=(0, 0.1)),
# Small gaussian blur with random sigma between 0 and 0.5.
# But we only blur about 50% of all images.
iaa.Sometimes(0.5, iaa.GaussianBlur(sigma=(0, 0.5))),
# Strengthen or weaken the contrast in each image.
iaa.ContrastNormalization((0.75, 1.5)),
# 先将图片从RGB变换到HSV,然后将H值增加10,然后再变换回RGB
iaa.WithColorspace(to_colorspace="HSV",
from_colorspace="RGB",
children=iaa.WithChannels(
2, iaa.Add((10, 50)))),
iaa.AverageBlur(k=((2, 5), (1, 3))),
iaa.SimplexNoiseAlpha(first=iaa.EdgeDetect((0.0, 0.2)),
second=iaa.ContrastNormalization(
(0.5, 2.0)),
per_channel=True),
# Add gaussian noise.
# For 50% of all images, we sample the noise once per pixel.
# For the other 50% of all images, we sample the noise per pixel AND
# channel. This can change the color (not only brightness) of the
# pixels.
iaa.ImpulseNoise(p=0.02),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.02 * 255), per_channel=0.5),
# Make some images brighter and some darker.
# In 20% of all cases, we sample the multiplier once per channel,
# which can end up changing the color of the images.
iaa.Multiply((0.8, 1.2), per_channel=0.2),
iaa.PerspectiveTransform(scale=0.06),
# # 图像扭曲
# iaa.PiecewiseAffine(scale=(0.01, 0.05)),
# Apply affine transformations to each image.
# Scale/zoom them, translate/move them, rotate them and shear them.
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-45, 45),
shear=(-8, 8))
},
random_order=True)
def __init__(self, images,
config,
shuffle=True,
jitter=True,
norm=None):
self.generator = None
self.images = images
self.config = config
self.shuffle = shuffle
self.jitter = jitter
self.norm = norm
self.image_counter = 0
ia.seed( 1 )
sometimes = lambda aug: iaa.Sometimes(0.8, aug)
# Here the augmentation is defined but not executed.
# Define our sequence of augmentation steps that will be applied to every image
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
self.aug_pipe = iaa.Sequential(
[
sometimes( iaa.Affine(
#scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.1, 0.1), "y": (-0.1, 0.1)}, # translate by -20 to +20 percent (per axis)
rotate=(-10, 10), # rotate by -45 to +45 degrees
#shear=(-5, 5), # shear by -16 to +16 degrees
#order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
#cval=(0, 255), # if mode is constant, use a cval between 0 and 255
#mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
mode = "edge"
)),
iaa.SomeOf((0, 5),
[
#sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
]),
# iaa.CoarseSalt(0.01, size_percent=(0.002, 0.01)),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
#iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges
#sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
#])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.1*255), per_channel=0), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0), # randomly remove up to 10% of the pixels
#iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
#iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-15, 15), per_channel=0), # change brightness of images (by -10 to 10 of original value)
iaa.Multiply((0.5, 1.5), per_channel=0), # change brightness of images (50-150% of original value)
iaa.ContrastNormalization((0.5, 2.0), per_channel=0), # improve or worsen the contrast
#iaa.Grayscale(alpha=(0.0, 1.0)),
#sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
#sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
])
],
random_order=True
)
if shuffle: np.random.shuffle(self.images)
def _create_augment_pipeline():
from imgaug import augmenters as iaa
# augmentors by https://github.com/aleju/imgaug
def sometimes(aug):
return iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
aug_pipe = iaa.Sequential(
[
# apply the following augmenters to most images
# iaa.Fliplr(0.5), # horizontally flip 50% of all images
# iaa.Flipud(0.2), # vertically flip 20% of all images
# sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
sometimes(
iaa.Affine(
# scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
# translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
# rotate=(-5, 5), # rotate by -45 to +45 degrees
# shear=(-5, 5), # shear by -16 to +16 degrees
# order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 5),
[
# sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0
)), # blur images with a sigma between 0 and 3.0
# blur image using local means with kernel sizes between 2 and 7
iaa.AverageBlur(k=(2, 7)),
# blur image using local medians with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)),
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
# iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges
# sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
# ])),
# add gaussian noise to images
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.OneOf([
# randomly remove up to 10% of the pixels
iaa.Dropout((0.01, 0.1), per_channel=0.5),
#iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
# iaa.Invert(0.05, per_channel=True), # invert color channels
# change brightness of images (by -10 to 10 of original value)
iaa.Add((-10, 10), per_channel=0.5),
# change brightness of images (50-150% of original value)
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# improve or worsen the contrast
iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),
#iaa.Grayscale(alpha=(0.0, 1.0)),
# sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
# sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
],
random_order=True)
],
random_order=True)
return aug_pipe
def train():
BATCH_SIZE = 100
network = Network()
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
# create directory for saving models
os.makedirs(os.path.join('save', network.description, timestamp))
dataset = Dataset(folder='data{}_{}'.format(network.IMAGE_HEIGHT,
network.IMAGE_WIDTH),
batch_size=BATCH_SIZE)
inputs, targets = dataset.next_batch()
print(inputs.shape, targets.shape)
# augmentation_seq = iaa.Sequential([
# iaa.Crop(px=(0, 16)), # crop images from each side by 0 to 16px (randomly chosen)
# iaa.Fliplr(0.5), # horizontally flip 50% of the images
# iaa.GaussianBlur(sigma=(0, 2.0)) # blur images with a sigma of 0 to 3.0
# ])
augmentation_seq = iaa.Sequential([
iaa.Crop(
px=(0, 16), name="Cropper"
), # crop images from each side by 0 to 16px (randomly chosen)
iaa.Fliplr(0.5, name="Flipper"),
iaa.GaussianBlur((0, 3.0), name="GaussianBlur"),
iaa.Dropout(0.02, name="Dropout"),
iaa.AdditiveGaussianNoise(scale=0.01 * 255, name="GaussianNoise"),
iaa.Affine(translate_px={
"x": (-network.IMAGE_HEIGHT // 3, network.IMAGE_WIDTH // 3)
},
name="Affine")
])
# change the activated augmenters for binary masks,
# we only want to execute horizontal crop, flip and affine transformation
def activator_binmasks(images, augmenter, parents, default):
if augmenter.name in ["GaussianBlur", "Dropout", "GaussianNoise"]:
return False
else:
# default value for all other augmenters
return default
hooks_binmasks = imgaug.HooksImages(activator=activator_binmasks)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter('{}/{}-{}'.format(
'logs', network.description, timestamp),
graph=tf.get_default_graph())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=None)
test_accuracies = []
# Fit all training data
n_epochs = 500
global_start = time.time()
for epoch_i in range(n_epochs):
dataset.reset_batch_pointer()
for batch_i in range(dataset.num_batches_in_epoch()):
batch_num = epoch_i * dataset.num_batches_in_epoch(
) + batch_i + 1
augmentation_seq_deterministic = augmentation_seq.to_deterministic(
)
start = time.time()
batch_inputs, batch_targets = dataset.next_batch()
batch_inputs = np.reshape(
batch_inputs, (dataset.batch_size, network.IMAGE_HEIGHT,
network.IMAGE_WIDTH, 1))
batch_targets = np.reshape(
batch_targets, (dataset.batch_size, network.IMAGE_HEIGHT,
network.IMAGE_WIDTH, 1))
batch_inputs = augmentation_seq_deterministic.augment_images(
batch_inputs)
batch_inputs = np.multiply(batch_inputs, 1.0 / 255)
batch_targets = augmentation_seq_deterministic.augment_images(
batch_targets, hooks=hooks_binmasks)
cost, _ = sess.run(
[network.cost, network.train_op],
feed_dict={
network.inputs: batch_inputs,
network.targets: batch_targets,
network.is_training: True
})
end = time.time()
print('{}/{}, epoch: {}, cost: {}, batch time: {}'.format(
batch_num, n_epochs * dataset.num_batches_in_epoch(),
epoch_i, cost, end - start))
if batch_num % 100 == 0 or batch_num == n_epochs * dataset.num_batches_in_epoch(
):
test_inputs, test_targets = dataset.test_set
# test_inputs, test_targets = test_inputs[:100], test_targets[:100]
test_inputs = np.reshape(
test_inputs,
(-1, network.IMAGE_HEIGHT, network.IMAGE_WIDTH, 1))
test_targets = np.reshape(
test_targets,
(-1, network.IMAGE_HEIGHT, network.IMAGE_WIDTH, 1))
test_inputs = np.multiply(test_inputs, 1.0 / 255)
print(test_inputs.shape)
summary, test_accuracy = sess.run(
[network.summaries, network.accuracy],
feed_dict={
network.inputs: test_inputs,
network.targets: test_targets,
network.is_training: False
})
summary_writer.add_summary(summary, batch_num)
print('Step {}, test accuracy: {}'.format(
batch_num, test_accuracy))
test_accuracies.append((test_accuracy, batch_num))
print("Accuracies in time: ", [
test_accuracies[x][0]
for x in range(len(test_accuracies))
])
max_acc = max(test_accuracies)
print("Best accuracy: {} in batch {}".format(
max_acc[0], max_acc[1]))
print("Total time: {}".format(time.time() - global_start))
# Plot example reconstructions
n_examples = 12
test_inputs, test_targets = dataset.test_inputs[:
n_examples], dataset.test_targets[:
n_examples]
test_inputs = np.multiply(test_inputs, 1.0 / 255)
test_segmentation = sess.run(
network.segmentation_result,
feed_dict={
network.inputs:
np.reshape(test_inputs, [
n_examples, network.IMAGE_HEIGHT,
network.IMAGE_WIDTH, 1
])
})
# Prepare the plot
test_plot_buf = draw_results(test_inputs, test_targets,
test_segmentation,
test_accuracy, network,
batch_num)
# Convert PNG buffer to TF image
image = tf.image.decode_png(test_plot_buf.getvalue(),
channels=4)
# Add the batch dimension
image = tf.expand_dims(image, 0)
# Add image summary
image_summary_op = tf.summary.image("plot", image)
image_summary = sess.run(image_summary_op)
summary_writer.add_summary(image_summary)
if test_accuracy >= max_acc[0]:
checkpoint_path = os.path.join('save',
network.description,
timestamp, 'model.ckpt')
saver.save(sess,
checkpoint_path,
global_step=batch_num)
except FileNotFoundError as e:
a = 1
mkdir(AUG_IMG_DIR)
AUGLOOP = 20 # 每张影像增强的数量
boxes_img_aug_list = []
new_bndbox = []
new_bndbox_list = []
# 影像增强
seq = iaa.Sequential([
iaa.Flipud(0.5), # vertically flip 20% of all images
iaa.Fliplr(0.5), # 镜像
iaa.Multiply((1.2, 1.5)), # change brightness, doesn't affect BBs
iaa.GaussianBlur(sigma=(0, 3.0)), # iaa.GaussianBlur(0.5),
iaa.Affine(
translate_px={
"x": 15,
"y": 15
},
scale=(0.8, 0.95),
rotate=(-30, 30)
) # translate by 40/60px on x/y axis, and scale to 50-70%, affects BBs
])
for root, sub_folders, files in os.walk(XML_DIR):
for name in files:
bndbox = read_xml_annotation(XML_DIR, name)
from PIL import Image
from keras.preprocessing.image import ImageDataGenerator
import keras
import pandas as pd
import numpy as np
import glob
import matplotlib.pyplot as plt
import imgaug.augmenters as iaa
#preparing data_augmentation generator
seq = iaa.SomeOf(
(0, 2),
[iaa.GaussianBlur(sigma=(0, 5)),
iaa.AdditiveGaussianNoise(scale=(0, 15))])
def imgaug(image):
return seq.augment_image(image)
train_gen = ImageDataGenerator(
rescale=1. / 255,
vertical_flip=1,
horizontal_flip=1,
rotation_range=360,
brightness_range=[0.3, 1.5],
# zca_whitening = True,
shear_range=10,
width_shift_range=.1,
height_shift_range=.1,
augmentation = iaa.Sequential([
iaa.OneOf([ ## geometric transform
iaa.Affine(
scale={"x": (0.98, 1.02), "y": (0.98, 1.04)},
translate_percent={"x": (-0.02, 0.02), "y": (-0.04, 0.04)},
rotate=(-2, 2),
shear=(-1, 1),
),
iaa.PiecewiseAffine(scale=(0.001, 0.025)),
]),
iaa.OneOf([ ## brightness or contrast
iaa.Multiply((0.9, 1.1)),
iaa.ContrastNormalization((0.9, 1.1)),
]),
iaa.OneOf([ ## blur or sharpen
iaa.GaussianBlur(sigma=(0.0, 0.1)),
iaa.Sharpen(alpha=(0.0, 0.1)),
]),
])
#
# # test on the same image as above
# imggrid = augmentation.draw_grid(image[:, :, 0], cols=5, rows=2)
# plt.figure(figsize=(30, 12))
# _ = plt.imshow(imggrid[:, :, 0], cmap='gray')
# plt.pause(5)
model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)
def limit_gpu_memory(memory_fraction, gpu_serial_number='0'):
config = tf.ConfigProto()
config.gpu_options.visible_device_list = gpu_serial_number
config.gpu_options.per_process_gpu_memory_fraction = memory_fraction
set_session(tf.Session(config=config))
seq = iaa.Sequential(
[
iaa.Fliplr(0.5), # horizontal flips
iaa.Flipud(0.5), # vertical flips
# iaa.Crop(percent=(0, 0.1)), # random crops
# Small gaussian blur with random sigma between 0 and 0.5.
# But we only blur about 50% of all images.
iaa.Sometimes(0.5, iaa.GaussianBlur(sigma=(0, 0.5))),
# Strengthen or weaken the contrast in each image.
iaa.ContrastNormalization((0.75, 1.5)),
# Add gaussian noise.
# For 50% of all images, we sample the noise once per pixel.
# For the other 50% of all images, we sample the noise per pixel AND
# channel. This can change the color (not only brightness) of the
# pixels.
iaa.Sometimes(
0.5, iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05 * 255))),
# Make some images brighter and some darker.
# In 20% of all cases, we sample the multiplier once per channel,
# which can end up changing the color of the images.
iaa.Multiply((0.8, 1.2))
# Apply affine transformations to each image.
# Scale/zoom them, translate/move them, rotate them and shear them.
