def soft(image_iteration):
iteration = image_iteration / (120 * 1.5)
frequency_factor = 0.05 + float(iteration) / 1200000.0
color_factor = float(iteration) / 1200000.0
dropout_factor = 0.198667 + (0.03856658 -
0.198667) / (1 +
(iteration / 196416.6)**1.863486)
blur_factor = 0.5 + (0.5 * iteration / 120000.0)
add_factor = 10 + 10 * iteration / 170000.0
multiply_factor_pos = 1 + (2.5 * iteration / 800000.0)
multiply_factor_neg = 1 - (0.91 * iteration / 800000.0)
contrast_factor_pos = 1 + (0.5 * iteration / 800000.0)
contrast_factor_neg = 1 - (0.5 * iteration / 800000.0)
#print ('iteration',iteration,'Augment Status ',frequency_factor,color_factor,dropout_factor,blur_factor,add_factor,
# multiply_factor_pos,multiply_factor_neg,contrast_factor_pos,contrast_factor_neg)
augmenter = iaa.Sequential(
[
iaa.Sometimes(frequency_factor, iaa.GaussianBlur(
(0, blur_factor))),
# blur images with a sigma between 0 and 1.5
iaa.Sometimes(
frequency_factor,
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, dropout_factor),
per_channel=color_factor)),
# add gaussian noise to images
iaa.Sometimes(
frequency_factor,
iaa.CoarseDropout((0.0, dropout_factor),
size_percent=(0.08, 0.2),
per_channel=color_factor)),
# randomly remove up to X% of the pixels
iaa.Sometimes(
frequency_factor,
iaa.Dropout((0.0, dropout_factor), per_channel=color_factor)),
# randomly remove up to X% of the pixels
iaa.Sometimes(
frequency_factor,
iaa.Add((-add_factor, add_factor), per_channel=color_factor)),
# change brightness of images (by -X to Y of original value)
iaa.Sometimes(
frequency_factor,
iaa.Multiply((multiply_factor_neg, multiply_factor_pos),
per_channel=color_factor)),
# change brightness of images (X-Y% of original value)
iaa.Sometimes(
frequency_factor,
iaa.ContrastNormalization(
(contrast_factor_neg, contrast_factor_pos),
per_channel=color_factor)),
# improve or worsen the contrast
iaa.Sometimes(frequency_factor, iaa.Grayscale(
(0.0, 1))), # put grayscale
],
random_order=True # do all of the above in random order
)
return augmenter
def getSeq(self):
self.sometimes = lambda aug: iaa.Sometimes(0.5, aug)
self.seq = iaa.Sequential(
[
# apply the following augmenters to most images
# iaa.Fliplr(0.5), # horizontally flip 50% of all images
# iaa.Flipud(0.5), # vertically flip 20% of all images
iaa.SomeOf(
(0, 5),
[
#sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0)
), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(
k=(2, 7)
), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(
k=(3, 11)
), # blur image using local medians with kernel sizes between 2 and 7
]),
#iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255),
per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
iaa.Invert(0.05,
per_channel=True), # invert color channels
iaa.Add(
(-10, 10), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation(
(-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply(
(0.5, 1.5), per_channel=True),
second=iaa.ContrastNormalization((0.5, 2.0)))
]),
iaa.ContrastNormalization(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
],
random_order=True)
],
random_order=True)
def data_aug(img, bboxs=None, keypoints=None, joint_nums=17):
'''
:param img: 需要进行数据增强的图像
:param bboxs: list, [ [x1, y1, x2, y2], ..., [xn1, yn1, xn2, yn2] ]
:param keypoints: 关键点, COCO format or Ai-challenger format, list of list, [ [num_joints x 3], [num_joints x 3], ..., ]
:return:
'''
is_flip = [random.randint(0, 1), random.randint(0, 1)]
seq = iaa.Sequential([
iaa.Affine(rotate=(-15, 15), scale=(0.8, 1.2), mode='constant'),
iaa.Multiply((0.7, 1.5)),
iaa.Grayscale(iap.Choice(a=[0, 1], p=[0.8, 0.2]),
from_colorspace='BGR'),
iaa.Fliplr(is_flip[0]),
iaa.Flipud(is_flip[1]),
])
seq_det = seq.to_deterministic()
bbs = None
kps = None
bbs_aug = None
kps_aug = None
# joint_nums = 14
new_bboxs = []
new_keypoints = []
kps_ori = copy.copy(keypoints)
kps_ori = np.reshape(np.asarray(kps_ori),
newshape=(-1, joint_nums,
3)) if kps_ori is not None else None
if bboxs is not None:
assert type(bboxs) == type([])
bbs = ia.BoundingBoxesOnImage([], shape=img.shape)
for box in bboxs:
bbs.bounding_boxes.append(
ia.BoundingBox(x1=box[0], y1=box[1], x2=box[2], y2=box[3]))
if keypoints is not None:
kps = ia.KeypointsOnImage([], shape=img.shape)
assert type(keypoints) == type([])
for single_person_keypoints in keypoints:
for i in range(joint_nums):
joint = single_person_keypoints[i * 3:i * 3 + 3]
kps.keypoints.append(ia.Keypoint(x=joint[0], y=joint[1]))
img_aug = seq_det.augment_image(img)
if bbs is not None:
# print(bbs)
bbs_aug = seq_det.augment_bounding_boxes([bbs])
# print(bbs_aug)
for i in range(len(bbs_aug[0].bounding_boxes)):
box_aug = bbs_aug[0].bounding_boxes[i]
box = [box_aug.x1, box_aug.y1, box_aug.x2, box_aug.y2]
new_bboxs.append(box)
if kps is not None:
# print(kps)
kps_aug = seq_det.augment_keypoints([kps])
for i in range(len(kps_aug[0].keypoints)):
point = kps_aug[0].keypoints[i]
new_keypoints.append([point.x, point.y, 1])
new_keypoints = np.reshape(np.asarray(new_keypoints),
newshape=(-1, joint_nums, 3))
# keep ori keypoint visiable attribute
for i in range(kps_ori.shape[0]):
for joint in range(kps_ori.shape[1]):
new_keypoints[i][joint][2] = kps_ori[i][joint][2]
if kps_ori[i][joint][0] == 0 or kps_ori[i][joint][1] == 0:
new_keypoints[i][joint] = np.asarray([0, 0, 0])
# if flip, change keypoint order (left <-> right)
# ai-format: [ 0-right_shoulder, 1-right_elbow, 2-right_wrist,
# 3-left_shoulder, 4-left_elbow, 5-left_wrist,
# 6-right_hip, 7-right_knee, 8-right_ankle,
# 9-left_hip, 10-left_knee, 11-left_ankle,
# 12-head, 13-neck ]
# coco-format: TODO add coco-foramt change index
# change_index = [[0, 3], [1, 4], [2, 5], [6, 9], [7, 10], [8, 11]]
change_index = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12],
[13, 14], [15, 16]]
for flip in is_flip:
if flip:
for i in range(kps_ori.shape[0]):
for index in change_index:
right_point = copy.copy(new_keypoints[i][index[0]])
new_keypoints[i][index[0]] = new_keypoints[i][index[1]]
new_keypoints[i][index[1]] = right_point
new_keypoints = [
list(np.reshape(single_person_keypoints, (-1, )))
for single_person_keypoints in new_keypoints
]
# test
# if bbs is not None:
# img_before = bbs.draw_on_image(img, color=(0, 255, 0), thickness=2)
# img_after = bbs_aug.draw_on_image(img_aug, color=(0,0,255), thickness=2)
# cv2.imshow('box ori', img_before)
# cv2.imshow('box after', img_after)
# cv2.waitKey(0)
# if kps is not None:
# img_before = kps.draw_on_image(img, color=(0, 255, 0), size=5)
# img_after = kps_aug.draw_on_image(img_aug, color=(0, 0, 255), size=5)
# for i in range(kps_ori.shape[0]):
# for joint in range(kps_ori.shape[1]):
# point = kps_ori[i][joint]
# cv2.putText(img_before, str(joint), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 250), 1)
# point = new_keypoints[i][3*joint:3*joint+3]
# # cv2.putText(img_after, str(point[2]), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 250), 1)
# cv2.putText(img_after, str(joint), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1,
# (0, 0, 250), 1)
# cv2.imshow('kps ori', img_before)
# cv2.imshow('kps after', img_after)
# cv2.waitKey(0)
return img_aug, new_bboxs, new_keypoints
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = WhaleDataset()
dataset_train.load_whale(args.dataset, "train")
dataset_train.prepare()
# Validation dataset
dataset_val = WhaleDataset()
dataset_val.load_whale(args.dataset, "val")
dataset_val.prepare()
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
# first run
#layers_training='heads'
# second run
#layers_training='3+'
# third run
layers_training = 'all'
epochs_to_train = 250
# adding image augmentation parameters
augmentation = iaa.Sometimes(
.667,
iaa.Sequential(
[
iaa.Fliplr(0.5), # horizontal flips
iaa.Crop(percent=(0, 0.1)), # random crops
# Small gaussian blur with random sigma between 0 and 0.25.
# But we only blur about 50% of all images.
iaa.Sometimes(0.5, iaa.GaussianBlur(sigma=(0, 0.25))),
# Strengthen or weaken the contrast in each image.
iaa.ContrastNormalization((0.75, 1.5)),
# Add gaussian noise.
# For 50% of all images, we sample the noise once per pixel.
# For the other 50% of all images, we sample the noise per pixel AND
# channel. This can change the color (not only brightness) of the
# pixels.
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05 * 255)),
# Make some images brighter and some darker.
# In 20% of all cases, we sample the multiplier once per channel,
# which can end up changing the color of the images.
iaa.Multiply((0.8, 1.2)),
# Apply affine transformations to each image.
# Scale/zoom them, translate/move them, rotate them and shear them.
iaa.Affine(
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
#translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
rotate=(-180, 180),
#shear=(-8, 8)
)
],
random_order=True)) # apply augmenters in random order
# old image aug parameters
"""
augmentation = iaa.Sometimes(0.9, [
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Multiply((0.8, 1.2)),
iaa.GaussianBlur(sigma=(0.0, 5.0))
])
"""
print("Training 5+ layerss with augmentation.")
print("*****Beginning training*****")
print("config.LEARNING_RATE", config.LEARNING_RATE)
print("layers_training:", layers_training)
print("epochs_to_train:", epochs_to_train)
print("augmentation: ", augmentation)
print("---")
print("Images: {}\nClasses: {}".format(len(dataset_train.image_ids),
dataset_train.class_names))
#print("Training network in its entirety with augmentation")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=epochs_to_train,
layers=layers_training,
augmentation=augmentation)
def get_augs():
iaa_Blur = iaa.OneOf([
iaa.GaussianBlur((0, 1.5)), # blur images with a sigma between 0 and 1.5
iaa.AverageBlur(k=(2, 4)), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 3 and 5
])
iaa_Sharpen = iaa.OneOf([
iaa.Sharpen(alpha=(0, 0.5), lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 0.5), strength=(0, 0.5)), # emboss images
])
iaa_Noise = iaa.OneOf([
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.03*255), per_channel=0.1), # add gaussian noise to images
iaa.Dropout((0, 0.02), per_channel=0.1), # randomly remove up to 10% of the pixels
#iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
])
iaa_Affine = iaa.OneOf([
iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
cval=(0, 255), # if mode is constant, use a cval between 0 and 255
mode='edge' # use zeros for padding
),
iaa.PiecewiseAffine(scale=(0.01, 0.05)), # move parts of the image around # is it fast?
iaa.PerspectiveTransform(scale=(0.01, 0.1)) # is it fast?
#iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25), # move pixels locally around (with random strengths) # we need borders
])
iaa_HSV = iaa.OneOf([
iaa.Grayscale(alpha=(0.0, 0.5)),
iaa.Add((-10, 10), per_channel=0.01), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation
iaa.Sequential([
iaa.ChangeColorspace(from_colorspace="RGB", to_colorspace="HSV"),
iaa.WithChannels(0, iaa.Add((-100, 100))),
iaa.ChangeColorspace(from_colorspace="HSV", to_colorspace="RGB")
]),
iaa.ContrastNormalization((0.7, 1.6), per_channel=0), # improve or worsen the contrast
iaa.Multiply((0.7, 1.2), per_channel=0.01),
iaa.FrequencyNoiseAlpha(
exponent=(-2, 2),
first=iaa.Multiply((0.85, 1.2), per_channel=False),
second=iaa.ContrastNormalization((0.5, 1.7))),
])
iaa_Simple = iaa.OneOf([
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.5), # vertically flip 20% of all images
iaa.CropAndPad(
percent=(-0.05, 0.1),
pad_mode='edge',
pad_cval=(0, 255)
),
])
augs = {
'simple': iaa_Simple,
'affine': iaa_Affine,
'hsv': iaa_HSV,
'sharpen': iaa_Sharpen,
'blur': iaa_Blur,
'noise': iaa_Noise,
}
return augs
def img_augment_run(input_path, output_path, coords_input, data_gen_args):
flatten = lambda l: [obj for sublist in l for obj in sublist]
img_list = glob.glob(os.path.join(input_path, "*.png"))
print(img_list)
coords = pd.read_csv(coords_input, header=[0, 1, 2], index_col=[0])
coords_aug = coords.copy()
for img_num, img_name in enumerate(img_list):
img = cv2.imread(img_name)
coord_row_x = coords.iloc[img_num, 0::2]
coord_row_y = coords.iloc[img_num, 1::2]
coords_list = [
Keypoint(float(x), float(y)) for [x, y] in zip(coord_row_x, coord_row_y)
]
keypoints = KeypointsOnImage(coords_list, shape=img.shape)
seq = iaa.Sequential(
[
iaa.Multiply(
(
0.5 - data_gen_args["brightness_range"],
0.5 + data_gen_args["brightness_range"],
)
),
iaa.Affine(
rotate=(
-1 * data_gen_args["rotation_range"],
data_gen_args["rotation_range"],
),
scale=(
1 - data_gen_args["zoom_range"],
1 + data_gen_args["zoom_range"],
),
shear=(
-1 * data_gen_args["shear_range"],
data_gen_args["shear_range"],
),
translate_percent={
"x": (
-1 * data_gen_args["width_shift_range"],
data_gen_args["width_shift_range"],
),
"y": (
-1 * data_gen_args["height_shift_range"],
data_gen_args["height_shift_range"],
),
},
),
]
)
# Augment keypoints and images
image_aug, kps_aug = seq(image=img, keypoints=keypoints)
coords_aug.iloc[img_num, :] = flatten(
[[kp.x, kp.y] for kp in kps_aug.keypoints]
)
idx_name = coords_aug.index[img_num]
idx_basename = os.path.basename(idx_name)
coords_aug.rename(
index={
idx_name: idx_name.replace(
idx_basename, "{}_aug.png".format(idx_basename.split(".")[0])
)
},
inplace=True,
)
print(
idx_name.replace(
idx_basename, "{}_aug.png".format(idx_basename.split(".")[0])
)
)
io.imsave(os.path.join(output_path, os.path.basename(img_name)), img)
io.imsave(
os.path.join(
output_path,
"{}_aug.png".format(os.path.basename(img_name).split(".")[0]),
),
image_aug,
)
# Adapted from DeepLabCut (to facilitate conversion to DLC-compatible format):
# https://github.com/DeepLabCut/DeepLabCut/blob/master/deeplabcut/generate_training_dataset/labeling_toolbox.py
coords_aug.sort_index(inplace=True)
coords_aug = coords_aug.append(coords)
coords_aug.to_csv(
os.path.join(
output_path, "{}.csv".format(os.path.basename(coords_input).split(".")[0])
)
)
coords_aug.to_hdf(
os.path.join(
output_path, "{}.h5".format(os.path.basename(coords_input).split(".")[0])
),
"df_with_missing",
format="table",
mode="w",
)
def img_aug_fuc(img, kps):
'''
function that used for data augmentation
:param img: ori_image that was prepared to run aug, shape must be [h, w, c]
:param kps: a ndarray that contains keypoints on this image. shape must be [person_num, joints_num, 3].
person_num means that there contains 'person_num' person on this image
joints_num means that we want to detect how many joints. e.g. like 1 for single head point or 14 for all body points in ai-challenger format.
3 means [x, y, v], v is the visable attribute for one joint point.
:return: img and kps after augmentation.
'''
kps_ori = np.copy(kps)
# 定义一个变换序列, 包括
# 1. 改变亮度
# 2. (-90, 90)内旋转并缩放
# 3. 随机变成灰度图
# 4. 随机水平翻转
# 5. 随机上下翻转
p_flip = np.random.randint(0, 2, (2))
# p_flip = [1, 0]
seq = iaa.Sequential([
iaa.Fliplr(p_flip[0]), # 50% 水平flip
iaa.Flipud(p_flip[1]), # 50% vertical flip
iaa.Multiply((0.7, 1.5)), # 改变亮度,不影响关键点 (后面可逐步增大至4, 5, 6)
iaa.Affine(rotate=(-45, 45), scale=(0.5, 0.7),
mode='constant'), # 旋转然后缩放
iaa.Grayscale((0.0, 1.0)), # 随机变成灰度图
])
seq_det = seq.to_deterministic()
# 定义keypoints
keypoints = ia.KeypointsOnImage([], shape=img.shape)
person_num, joints_num, _ = kps.shape
for person in range(person_num):
for joint in range(joints_num):
point = kps[person][joint]
keypoints.keypoints.append(ia.Keypoint(x=point[0], y=point[1]))
# 执行augmentation
img_aug = seq_det.augment_image(img).copy()
kps_aug = seq_det.augment_keypoints([keypoints])[0]
# 返回augmentation之后的keypoints
ret_kps = []
for i in range(len(keypoints.keypoints)):
point_after = kps_aug.keypoints[i]
ret_kps.append([point_after.x, point_after.y, 1])
ret_kps = np.reshape(np.asarray(ret_kps), newshape=(-1, joints_num, 3))
assert img_aug.shape == img.shape
assert ret_kps.shape == kps_ori.shape
# keep ori keypoint visiable attribute
for person in range(ret_kps.shape[0]):
for joint in range(ret_kps.shape[1]):
ret_kps[person][joint][2] = int(kps_ori[person][joint][2])
# import cv2
# img_test = img_aug.copy()
# for person in range(ret_kps.shape[0]):
# for joint in range(ret_kps.shape[1]):
# point = ret_kps[person][joint]
# cv2.circle(img_test, (int(point[0]), int(point[1])), 2, (255, 0, 0), -1)
# cv2.putText(img_test, str(joint), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('after 0 ', img_test)
# cv2.waitKey(0)
if joints_num > 2:
import copy
# exchange left and right point
# [ 0-right_shoulder, 1-right_elbow, 2-right_wrist, 3-left_shoulder, 4-left_elbow, 5-left_wrist,
# 6-right_hip, 7-right_knee, 8-right_ankle, 9-left_hip, 10-left_knee, 11-left_ankle, 12-head, 13-neck ]
change_index = [[0, 3], [1, 4], [2, 5], [6, 9], [7, 10], [8, 11]]
for p in p_flip:
if p == 1:
for person in range(ret_kps.shape[0]):
for index in change_index:
left_point = copy.copy(ret_kps[person][index[0]])
ret_kps[person][index[0]] = ret_kps[person][index[1]]
ret_kps[person][index[1]] = left_point
# image_before = keypoints.draw_on_image(img, size=7)
# # image_after = kps_aug.draw_on_image(img_aug, size=7)
# for person in range(ret_kps.shape[0]):
# for joint in range(ret_kps.shape[1]):
# point = kps_ori[person][joint]
# # print (point)
# cv2.circle(img, (int(point[0]), int(point[1])), 2, (255, 0, 0), -1)
# cv2.putText(img, str(joint) + str(point[2]), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
#
# cv2.imshow('before', img)
# cv2.waitKey(0)
# for person in range(ret_kps.shape[0]):
# for joint in range(ret_kps.shape[1]):
# point = ret_kps[person][joint]
# # print (point)
# cv2.circle(img_aug, (int(point[0]), int(point[1])), 2, (255, 0, 0), -1)
# cv2.putText(img_aug, str(point[2]), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('after', img_aug)
# cv2.waitKey(0)
# print (ret_kps)
return img_aug, ret_kps
# In[ ]:
# Image augmentation
augmentation = iaa.SomeOf((0, 1), [
iaa.Fliplr(0.5),
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-25, 25),
shear=(-8, 8)),
iaa.Multiply((0.9, 1.1))
])
# ### Now it's time to train the model. Note that training even a basic model can take a few hours.
#
# Note: the following model is for demonstration purpose only. We have limited the training to one epoch, and have set nominal values for the Detector Configuration to reduce run-time.
#
# - dataset_train and dataset_val are derived from DetectorDataset
# - DetectorDataset loads images from image filenames and masks from the annotation data
# - model is Mask-RCNN
# In[ ]:
NUM_EPOCHS = 1
# Train Mask-RCNN Model
def main():
IN_JSON = 'dataset_descriptions/iamondb_generated_dates_10k.json'
OUT_JSON = 'dataset_descriptions/iamondb_generated_dates_resized_aug2_10k.json'
IMG_IN_DIR = 'datasets/tars'
IMG_OUT_DIR = 'datasets/'
SCALE_FACTOR = 2 # How much bigger the new dataset should be
SAVE_ORIGINAL = False
TARGET_DIMENSIONS = (216, 64)
new_image_infos = []
with open(IN_JSON, 'r') as json_file:
json_images = json.load(json_file)
image_infos = copy.deepcopy(json_images)
for idx in range(0, len(image_infos)):
image_infos[idx]['path'] = os.path.join(
IMG_IN_DIR, os.path.basename(image_infos[idx]['path']))
seq = iaa.Sequential(
[
iaa.Sometimes(0.5, iaa.GaussianBlur(
sigma=(0, 0.5))), # Small blur on half of the images
iaa.ContrastNormalization((0.75, 1.5)), # Change in contrast
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255),
per_channel=0.5), # Gaussian Noise can change colour
iaa.Multiply((0.8, 1.2), per_channel=0.2) #, # brighter/darker
# iaa.Affine(
# rotate=(-10, 10),
# shear=(-20, 20),
# fit_output=True,
# ),
# iaa.PiecewiseAffine(scale=(0.01, 0.05), mode='edge')
],
random_order=True) # apply augmenters in random order
images = np.array(
[np.array(Image.open(img['path'])) for img in image_infos])
images = np.repeat(images, SCALE_FACTOR, axis=0)
image_infos = np.repeat(np.array(image_infos), SCALE_FACTOR, axis=0)
# Make batches out of single images because processing multiple images at once leads to errors because the resulting
# images have different shapes and therefore can't be organised in a numpy array -> leads to an internal error
for idx, img in enumerate(images):
old_filename = os.path.splitext(
os.path.basename(image_infos[idx]['path']))[0]
new_filename = old_filename + '-' + str(idx % SCALE_FACTOR +
1) + '.png'
new_img_path = os.path.join(IMG_OUT_DIR, new_filename)
if SAVE_ORIGINAL and idx % SCALE_FACTOR == 0:
with open(os.path.join(IMG_OUT_DIR, old_filename + '.png'),
'wb+') as orig_img_file:
resize_img(Image.fromarray(img),
TARGET_DIMENSIONS).save(orig_img_file, format='PNG')
padded_img = resize_img(Image.fromarray(img),
TARGET_DIMENSIONS,
padding_color=255)
img_aug = seq(images=np.expand_dims(np.asarray(padded_img), axis=0))
pil_img = Image.fromarray(np.squeeze(img_aug)).convert('L')
with open(new_img_path, 'wb+') as img_file:
pil_img.save(img_file, format='PNG')
new_image_infos.append({
"string": image_infos[idx]['string'],
"type": image_infos[idx]['type'],
"path": new_img_path
})
with open(OUT_JSON, 'w') as out_json:
if SAVE_ORIGINAL:
assert os.path.split(json_images[0]['path'])[0] == os.path.split(new_image_infos[0]['path'])[0],\
"Original path differs from new path"
json.dump(json_images + new_image_infos,
out_json,
ensure_ascii=False,
indent=4)
else:
json.dump(new_image_infos, out_json, ensure_ascii=False, indent=4)
print(
f'Created {len(new_image_infos)} new images (original: {len(json_images)})'
)
print("Image area <= {:4}**2: None".format(np.sqrt(image_area)))
continue
print(
"Image area <= {:4.0f}**2: mean: {:.1f} median: {:.1f} min: {:.1f} max: {:.1f}"
.format(np.sqrt(image_area), tumor_per_image.mean(),
np.median(tumor_per_image), tumor_per_image.min(),
tumor_per_image.max()))
ax[i].set_title("Image Area <= {:4}**2".format(np.sqrt(image_area)))
_ = ax[i].hist(tumor_per_image, bins=10)
# List of augmentations
# http://imgaug.readthedocs.io/en/latest/source/augmenters.html
augmentation = iaa.Sometimes(0.9, [
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Multiply((0.8, 1.2)),
iaa.GaussianBlur(sigma=(0.0, 5.0))
])
# Load the image multiple times to show augmentations
limit = 4
ax = get_ax(rows=2, cols=limit // 2)
for i in range(limit):
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset,
config,
image_id,
use_mini_mask=False,
augment=True,
augmentation=augmentation)
visualize.display_instances(image,
def get_default_imgaug():
# Возвращает набор аугментаций imgaug'а, который по нашей договоренности является дефолтным (наиболее используемым)
# Набор аугментаций уже непосредствено можно применять к данным
# Вызывается конструктором DataGenerator, если в конструктор DataGenerator не передан свой набор аугментаций.
# По большей части повторяет пример из документации: https://imgaug.readthedocs.io/en/latest/source/examples_basics.html#a-simple-and-common-augmentation-sequence
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
return iaa.Sequential(
[
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
sometimes(iaa.Crop(percent=(0, 0.1))),
sometimes(
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-45, 45),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL)),
iaa.SomeOf(
(0, 5),
[
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
sometimes(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(alpha=(0, 0.7),
direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
iaa.Invert(0.05,
per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
#iaa.LinearContrast((0.5, 2.0), per_channel=0.5), ПОЧЕМУ ТО НЕ РОБИТ
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True)
],
# do all of the above augmentations in random order
random_order=True)
iaa.BlendAlphaSimplexNoise(iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.BlendAlphaFrequencyNoise(
exponent=(-4, 0),
foreground=iaa.Multiply((0.5, 1.5), per_channel=True),
background=iaa.LinearContrast((0.5, 2.0))
)
]),
iaa.LinearContrast((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True
)
],
def get_example(self, i):
files_dir = self._files_dirs[i]
image_file = osp.join(files_dir, 'image.png')
image = skimage.io.imread(image_file)
assert image.dtype == np.uint8
assert image.ndim == 3
depth_file = osp.join(files_dir, 'depth.npz')
depth = np.load(depth_file)['arr_0']
depth[depth == 0] = np.nan
if depth.dtype == np.uint16:
depth = depth.astype(np.float32)
depth /= 1000
depth_keep = ~np.isnan(depth)
depth[depth_keep] = np.maximum(depth[depth_keep], self.min_value)
depth[depth_keep] = np.minimum(depth[depth_keep], self.max_value)
assert depth.dtype == np.float32
assert depth.ndim == 2
label_file = osp.join(files_dir, 'label.png')
with open(label_file, 'r') as f:
label = np.asarray(PIL.Image.open(f)).astype(np.int32)
assert label.dtype == np.int32
assert label.ndim == 2
depth_gt_file = osp.join(files_dir, 'depth_gt.npz')
depth_gt = np.load(depth_gt_file)['arr_0']
depth_gt[depth_gt == 0] = np.nan
if depth_gt.dtype == np.uint16:
depth_gt = depth_gt.astype(np.float32)
depth /= 1000
depth_gt_keep = ~np.isnan(depth_gt)
depth_gt[depth_gt_keep] = np.maximum(depth_gt[depth_gt_keep],
self.min_value)
depth_gt[depth_gt_keep] = np.minimum(depth_gt[depth_gt_keep],
self.max_value)
assert depth_gt.dtype == np.float32
assert depth_gt.ndim == 2
# Data augmentation
if self.aug:
# 1. Color augmentation
obj_datum = dict(img=image)
random_state = np.random.RandomState()
def st(x):
return iaa.Sometimes(0.3, x)
augs = [
st(iaa.Add([-50, 50], per_channel=True)),
st(
iaa.InColorspace('HSV',
children=iaa.WithChannels(
[1, 2], iaa.Multiply([0.5, 2])))),
st(iaa.GaussianBlur(sigma=[0.0, 1.0])),
st(
iaa.AdditiveGaussianNoise(scale=(0.0, 0.1 * 255),
per_channel=True)),
]
obj_datum = next(
mvtk.aug.augment_object_data([obj_datum],
random_state=random_state,
augmentations=augs))
image = obj_datum['img']
# 2. Depth noise
np.random.seed()
if np.random.uniform() < 0.3:
noise_rate = np.random.uniform() * 0.25 + 0.05
depth[np.random.rand(depth.shape[0], depth.shape[1]) <
noise_rate] = np.nan
# 3. Geometric augmentation
if np.random.uniform() < 0.5:
image = np.fliplr(image)
depth = np.fliplr(depth)
label = np.fliplr(label)
depth_gt = np.fliplr(depth_gt)
if np.random.uniform() < 0.5:
image = np.flipud(image)
depth = np.flipud(depth)
label = np.flipud(label)
depth_gt = np.flipud(depth_gt)
if np.random.uniform() < 0.5:
angle = (np.random.uniform() * 180) - 90
image = self.rotate_image(image, angle, cv2.INTER_LINEAR)
depth = self.rotate_depth_image(depth, angle, cv2.INTER_LINEAR)
label = self.rotate_image(label, angle, cv2.INTER_NEAREST)
depth_gt = self.rotate_depth_image(depth_gt, angle,
cv2.INTER_LINEAR)
return image, depth, label, depth_gt
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = Deep_Lesion_Dataset()
############################## We need to separate the training and validation
### There is a variable in the dict, which is training_val_testing
### It's in image_info file,
# under annotations, image
#annotations['images'][0]['Train_Val_Test']
### TRAIN = 1
### VALIDATION = 2
### TEST = 3
###### NEED TO EXRACT THREE DIFFERENT DATASETS based on this.
## look at how "train"/"valid" is fed in below
dataset_train.load_deep_lesion(args.dataset, "train")
dataset_train.prepare()
# Validation dataset
dataset_val = Deep_Lesion_Dataset()
dataset_val.load_deep_lesion(args.dataset, "val")
dataset_val.prepare()
augmentation = iaa.SomeOf(
(0, 1),
[
#iaa.Fliplr(0.5),
iaa.Affine(scale={
"x": (0.9, 1.1),
"y": (0.9, 1.1)
},
translate_percent={
"x": (-0.1, 0.1),
"y": (-0.1, 0.1)
},
rotate=(-5, 5),
shear=(-1, 1)),
iaa.Multiply((0.9, 1.1))
])
########################################################################################
from keras.callbacks import ReduceLROnPlateau
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.3,
patience=patience1,
verbose=1,
mode='min',
cooldown=0,
min_lr=0)
callbacks = [reduce_lr]
print("Training Images: {}\nClasses: {}".format(
len(dataset_train.image_ids), dataset_train.class_names))
print("Validations Images: {}\nClasses: {}".format(
len(dataset_val.image_ids), dataset_val.class_names))
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
print("Training network heads")
model.train(
dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
#epochs=30,
#layers='heads')
epochs=20,
layers='all',
custom_callbacks=callbacks, #############
augmentation=augmentation) ######################
print("finised training network")
iaa.Dropout((0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add(
(-10, 10), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation(
(-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(exponent=(-4, 0),
first=iaa.Multiply(
(0.5, 1.5), per_channel=True),
second=iaa.LinearContrast(
(0.5, 2.0)))
]),
iaa.LinearContrast(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
), # move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(
0.01, 0.05))), # sometimes move parts of the image around
def imgaug_img(img):
# Sometimes(0.5, ...) applies the given augmenter in 50% of all cases,
# e.g. Sometimes(0.5, GaussianBlur(0.3)) would blur roughly every second
# image.
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image.
seq = iaa.Sequential(
[
#
# Apply the following augmenters to most images.
#
iaa.Fliplr(0.5), # horizontally flip 50% of all images
# iaa.Flipud(0.2), # vertically flip 20% of all images
# crop some of the images by 0-10% of their height/width
sometimes(iaa.Crop(percent=(0, 0.1))),
# Apply affine transformations to some of the images
# - scale to 80-120% of image height/width (each axis independently)
# - translate by -20 to +20 relative to height/width (per axis)
# - rotate by -45 to +45 degrees
# - shear by -16 to +16 degrees
# - order: use nearest neighbour or bilinear interpolation (fast)
# - mode: use any available mode to fill newly created pixels
# see API or scikit-image for which modes are available
# - cval: if the mode is constant, then use a random brightness
# for the newly created pixels (e.g. sometimes black,
# sometimes white)
sometimes(
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-45, 45),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL)),
#
# Execute 0 to 5 of the following (less important) augmenters per
# image. Don't execute all of them, as that would often be way too
# strong.
#
iaa.SomeOf(
(0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
sometimes(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(alpha=(0, 0.7),
direction=(0.0, 1.0)),
])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
iaa.Invert(0.05,
per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.LinearContrast((0.5, 2.0), per_channel=0.5),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True)
],
# do all of the above augmentations in random order
random_order=True)
images_aug = seq.augment_images([img])
return images_aug[0]
meta = {'noop': iaa.Noop(), 'shuffle': iaa.ChannelShuffle(p=1.0)}
arithmetic = {
'add-45': iaa.Add(value=-45),
'add-25': iaa.Add(value=-25),
'add+25': iaa.Add(value=25),
'add+45': iaa.Add(value=45),
'addp-': iaa.Add(value=(-35, -15), per_channel=True),
'addp+': iaa.Add(value=(15, 35), per_channel=True),
'addGN': iaa.AdditiveGaussianNoise(scale=0.10 * 255),
'addGNp': iaa.AdditiveGaussianNoise(scale=0.10 * 255, per_channel=True),
'addLN': iaa.AdditiveLaplaceNoise(scale=0.10 * 255),
'addLNp': iaa.AdditiveLaplaceNoise(scale=0.10 * 255, per_channel=True),
'addPN': iaa.AdditivePoissonNoise(lam=16.00),
'addPNp': iaa.AdditivePoissonNoise(lam=16.00, per_channel=True),
'mul-': iaa.Multiply(mul=0.50),
'mul+': iaa.Multiply(mul=1.50),
'mulp-': iaa.Multiply(mul=0.50, per_channel=True),
'mulp+': iaa.Multiply(mul=1.50, per_channel=True),
'jpeg': iaa.JpegCompression(compression=62),
'jpeg+': iaa.JpegCompression(compression=75),
'jpeg++': iaa.JpegCompression(compression=87)
}
blur = {
'GBlur': iaa.GaussianBlur(sigma=1.00),
'ABlur': iaa.AverageBlur(k=3),
'MBlur': iaa.MedianBlur(k=3),
'BBlur': iaa.BilateralBlur(sigma_color=250, sigma_space=250, d=5),
'MoBlur': iaa.MotionBlur(angle=0, k=7),
'MoBlurAng': iaa.MotionBlur(angle=144, k=5)
def main():
augmenters = [
iaa.Identity(name="Identity"),
iaa.Crop(px=(0, 8), name="Crop-px"),
iaa.Crop(percent=(0, 0.1), name="Crop-percent"),
iaa.Fliplr(0.5, name="Fliplr"),
iaa.Flipud(0.5, name="Flipud"),
iaa.Grayscale((0.0, 1.0), name="Grayscale"),
iaa.GaussianBlur((0, 3.0), name="GaussianBlur"),
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, 0.1),
name="AdditiveGaussianNoise"),
iaa.Dropout((0.0, 0.1), name="Dropout"),
iaa.Multiply((0.5, 1.5), name="Multiply"),
iaa.ContrastNormalization(alpha=(0.5, 2.0),
name="ContrastNormalization"),
iaa.Grayscale(alpha=(0.0, 1.0), name="Grayscale"),
iaa.ElasticTransformation(alpha=(0.5, 8.0),
sigma=1.0,
name="ElasticTransformation"),
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_px={
"x": (-16, 16),
"y": (-16, 16)
},
rotate=(-45, 45),
shear=(-16, 16),
order=0,
cval=(0, 255),
mode="constant",
name="AffineOrder0ModeConstant")
]
for order in [0, 1]:
augmenters.append(
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_px={
"x": (-16, 16),
"y": (-16, 16)
},
rotate=(-45, 45),
shear=(-16, 16),
order=order,
cval=(0, 255),
mode=ia.ALL,
name="AffineOrder%d" % (order, )))
augmenters.append(
iaa.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
translate_px={
"x": (-16, 16),
"y": (-16, 16)
},
rotate=(-45, 45),
shear=(-16, 16),
order=ia.ALL,
cval=(0, 255),
mode=ia.ALL,
name="AffineAll"))
kps = []
for _ in sm.xrange(20):
x = random.randint(0, 31)
y = random.randint(0, 31)
kps.append(ia.Keypoint(x=x, y=y))
kps = ia.KeypointsOnImage(kps, shape=(32, 32, 3))
small_keypoints_one = kps.on((4, 4, 3))
medium_keypoints_one = kps.on((32, 32, 3))
large_keypoints_one = kps.on((256, 256, 3))
small_keypoints = [small_keypoints_one.deepcopy() for _ in sm.xrange(16)]
medium_keypoints = [medium_keypoints_one.deepcopy() for _ in sm.xrange(16)]
large_keypoints = [large_keypoints_one.deepcopy() for _ in sm.xrange(16)]
small_images = np.random.randint(0, 255, (16, 4, 4, 3)).astype(np.uint8)
medium_images = np.random.randint(0, 255, (16, 32, 32, 3)).astype(np.uint8)
large_images = np.random.randint(0, 255,
(16, 256, 256, 3)).astype(np.uint8)
print("---------------------------")
print("Keypoints")
print("---------------------------")
for augmenter in augmenters:
print("[Augmenter: %s]" % (augmenter.name, ))
for keypoints in [small_keypoints, medium_keypoints, large_keypoints]:
times = []
for i in sm.xrange(100):
time_start = time.time()
_img_aug = augmenter.augment_keypoints(keypoints)
time_end = time.time()
times.append(time_end - time_start)
times = np.array(times)
img_str = "{:20s}".format(keypoints[0].shape)
print("%s | SUM %.5fs | PER ITER avg %.5fs, min %.5fs, max %.5fs" %
(img_str, float(np.sum(times)), np.average(times),
np.min(times), np.max(times)))
print("---------------------------")
print("Images")
print("---------------------------")
for augmenter in augmenters:
print("[Augmenter: %s]" % (augmenter.name, ))
for images in [small_images, medium_images, large_images]:
times = []
for i in sm.xrange(100):
time_start = time.time()
_img_aug = augmenter.augment_images(images)
time_end = time.time()
times.append(time_end - time_start)
times = np.array(times)
img_str = "{:20s}".format(images.shape)
print("%s | SUM %.5fs | PER ITER avg %.5fs, min %.5fs, max %.5fs" %
(img_str, float(np.sum(times)), np.average(times),
np.min(times), np.max(times)))
from imgaug import augmenters as iaa
warnings.simplefilter('ignore', category=NumbaDeprecationWarning)
warnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning)
seq = iaa.Sequential([
iaa.Resize(config.resize_wh),
iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Crop(percent=(0, 0.2)),
iaa.Sometimes(
0.5, iaa.GaussianBlur(sigma=(0, 0.1)),
iaa.CoarseDropout(
(0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2)),
iaa.ContrastNormalization((0.75, 1.5)),
iaa.Multiply((0.8, 1.2), per_channel=0.2),
iaa.Affine(scale={
"x": (0.9, 1.1),
"y": (0.9, 1.1)
},
translate_percent={
"x": (-0.1, 0.1),
"y": (-0.1, 0.1)
},
rotate=(-25, 25),
shear=0)
],
random_order=True)
],
random_order=False)
def RandMultiply(img):
return iaa.Multiply((0.9, 1.1)).augment_image(img)
def _load_augmentation_aug_all():
""" Load image augmentation model """
def sometimes(aug):
return iaa.Sometimes(0.5, aug)
return iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop images by -5% to 10% of their height/width
sometimes(
iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode='constant',
pad_cval=(0, 255))),
sometimes(
iaa.Affine(
# scale images to 80-120% of their size, individually per axis
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
# translate by -20 to +20 percent (per axis)
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
# use nearest neighbour or bilinear interpolation (fast)
order=[0, 1],
# if mode is constant, use a cval between 0 and 255
cval=(0, 255),
# use any of scikit-image's warping modes
# (see 2nd image from the top for examples)
mode='constant')),
# execute 0 to 5 of the following (less important) augmenters per
# image don't execute all of them, as that would often be way too
# strong
iaa.SomeOf(
(0, 5),
[
# convert images into their superpixel representation
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
iaa.OneOf([
# blur images with a sigma between 0 and 3.0
iaa.GaussianBlur((0, 3.0)),
# blur image using local means with kernel sizes
# between 2 and 7
iaa.AverageBlur(k=(2, 7)),
# blur image using local medians with kernel sizes
# between 2 and 7
iaa.MedianBlur(k=(3, 11)),
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.BlendAlphaSimplexNoise(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
# add gaussian noise to images
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
iaa.OneOf([
# randomly remove up to 10% of the pixels
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
# invert color channels
iaa.Invert(0.05, per_channel=True),
# change brightness of images (by -10 to 10 of original value)
iaa.Add((-10, 10), per_channel=0.5),
# change hue and saturation
iaa.AddToHueAndSaturation((-20, 20)),
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.BlendAlphaFrequencyNoise(
exponent=(-4, 0),
foreground=iaa.Multiply(
(0.5, 1.5), per_channel=True),
background=iaa.contrast.LinearContrast((0.5, 2.0)))
]),
# improve or worsen the contrast
iaa.contrast.LinearContrast((0.5, 2.0), per_channel=0.5),
iaa.Grayscale(alpha=(0.0, 1.0)),
# move pixels locally around (with random strengths)
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# sometimes move parts of the image around
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
],
random_order=True)
shutil.rmtree(AUG_XML_DIR)
except FileNotFoundError as e:
a = 1
mkdir(AUG_XML_DIR)
AUGLOOP = 15 # 每张影像增强的数量
boxes_img_aug_list = []
new_bndbox = []
new_bndbox_list = []
# 影像增强
seq = iaa.Sequential([
iaa.Flipud(0.5), # vertically flip 20% of all images
iaa.Fliplr(0.5), # 镜像
iaa.Multiply((1.2, 1.5)), # change brightness, doesn't affect BBs
iaa.GaussianBlur(sigma=(0, 3.0)), # iaa.GaussianBlur(0.5),
iaa.Affine(
translate_px={
"x": 15,
"y": 15
},
scale=(0.8, 0.95),
rotate=(-30, 30)
) # translate by 40/60px on x/y axis, and scale to 50-70%, affects BBs
])
for root, sub_folders, files in os.walk(XML_DIR):
for name in files:
def img_random_brightness(image): brightness = iaa.Multiply((0.2, 1.2)) image = brightness.augment_image(image) return image
def __init__(self):
configMain.__init__(self)
st = lambda aug: iaa.Sometimes(0.4, aug)
oc = lambda aug: iaa.Sometimes(0.3, aug)
rl = lambda aug: iaa.Sometimes(0.09, aug)
self.augment = iaa.Sequential([
rl(iaa.GaussianBlur((0, 1.5))), # blur images with a sigma between 0 and 1.5
rl(iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05), per_channel=0.5)), # add gaussian noise to images
oc(iaa.Dropout((0.0, 0.10), per_channel=0.5)), # randomly remove up to X% of the pixels
oc(iaa.CoarseDropout((0.0, 0.10), size_percent=(0.08, 0.2),per_channel=0.5)), # randomly remove up to X% of the pixels
oc(iaa.Add((-40, 40), per_channel=0.5)), # change brightness of images (by -X to Y of original value)
st(iaa.Multiply((0.10, 2.5), per_channel=0.2)), # change brightness of images (X-Y% of original value)
rl(iaa.ContrastNormalization((0.5, 1.5), per_channel=0.5)), # improve or worsen the contrast
rl(iaa.Grayscale((0.0, 1))), # put grayscale
],
random_order=True # do all of the above in random order
)
self.augment_labels = True
self.augment_amount = 2 #3=max, 2=mid, 1=min
self.labels_to_augment = {"road": True, "buildings": True, "grass": True, "sky_n_zebra": True }
# there are files with data, 200 images each, and here we select which ones to use
#5self.dataset_name = 'Carla'
#with open(os.path.join(self.save_data_stats, 'path'),'r') as f:
# path = f.read().strip()
path = '../VirtualElektraData2_Double'
train_path = os.path.join(path, 'SeqTrain')
val_path = os.path.join(path, 'SeqVal')
print train_path, val_path
self.train_db_path = [os.path.join(train_path, f) for f in glob.glob1(train_path, "data_*.h5")]
self.val_db_path = [os.path.join(val_path, f) for f in glob.glob1(val_path, "data_*.h5")]
# When using data with noise, remove the recording during the first half of the noise impulse
# TODO Felipe: change to noise percentage.
self.remove_noise = False
# Speed Divide Factor
#TODO: FOr now is hardcooded, but eventually we should be able to calculate this from data at the loading time.
self.speed_factor = 1.0 # In KM/H FOR GTA it should be maximun 30.0
# The division is made by three diferent data kinds
# in every mini-batch there will be equal number of samples with labels from each group
# e.g. for [[0,1],[2]] there will be 50% samples with labels 0 and 1, and 50% samples with label 2
self.labels_per_division = [[2],[2],[2]]
self.dataset_names = ['targets']
self.queue_capacity = 20*self.batch_size
# TODO NOT IMPLEMENTED Felipe: True/False switches to turn data balancing on or off
self.balances_val = True
self.balances_train = True
self.augment_and_saturate_factor = True
def augmentation_generator(data, batch_size):
'''
Performs data augmentation on training set. Practice data gen.
'''
n = len(data)
batches_to_make = n // batch_size
# Containers for batch data and labels.
batch_data = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
batch_labels = np.zeros((batch_size, 2), dtype=np.float32)
# Grab indices.
indices = np.arange(n)
# Define transformations.
transformations = iaa.OneOf([
iaa.Fliplr(0.5), # Flip.
iaa.ContrastNormalization((0.75, 1.5)), # Play around with contrast.
iaa.GaussianBlur(sigma=(0, 0.5)), # Randomly blur.
iaa.Multiply((0.8, 1.2)) # Change brightness.
])
# Start a counter.
batches_made = 0
while True:
# Make batch.
np.random.shuffle(indices)
count = 0
next_batch = indices[(batches_made * batch_size):(batches_made + 1) *
batch_size]
for foo, idx in enumerate(next_batch):
img_name = data.iloc[idx]['image_path']
img_label = data.iloc[idx]['label']
# Encode label.
encoded_label = to_categorical(img_label, num_classes=2)
# Read image and resize.
img = cv2.imread(str(img_name))
img = cv2.resize(img, (224, 224))
# Check for greyscale.
if img.shape[2] == 1:
img = np.dstack([img, img, img])
# Change from BGR to RGB and normalize.
orig_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
orig_img = orig_img.astype(np.float32) / 255
batch_data[count] = orig_img
batch_labels[count] = encoded_label
# Data augment for under-sampled class.
if img_label == 0 and count < batch_size - 2:
# Image 1.
aug_img1 = transformations.augment(images=img)
aug_img1 = cv2.cvtColor(aug_img1, cv2.COLOR_BGR2RGB)
aug_img1 = aug_img1.astype(np.float32) / 255
batch_data[count + 1] = aug_img1
batch_labels[count + 1] = encoded_label
# Image 2.
aug_img2 = transformations.augment(images=img)
aug_img2 = cv2.cvtColor(aug_img2, cv2.COLOR_BGR2RGB)
aug_img2 = aug_img2.astype(np.float32) / 255
batch_data[count + 2] = aug_img2
batch_labels[count + 2] = encoded_label
else:
count += 1
# Can't add anymore.
if count == batch_size - 1:
break
batches_made += 1
yield batch_data, batch_labels
# Reset for the next go.
if batches_made >= batches_to_make:
batches_made = 0
def __init__(self, images, config, shuffle=True, jitter=True, norm=None):
self.generator = None
self.images = images
self.config = config
self.shuffle = shuffle
self.jitter = jitter
self.norm = norm
self.anchors = [
BoundBox(0, 0, config['ANCHORS'][2 * i],
config['ANCHORS'][2 * i + 1])
for i in range(int(len(config['ANCHORS']) // 2))
]
### augmentors by https://github.com/aleju/imgaug
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image
# All augmenters with per_channel=0.5 will sample one value _per image_
# in 50% of all cases. In all other cases they will sample new values
# _per channel_.
self.aug_pipe = iaa.Sequential(
[
# apply the following augmenters to most images
#iaa.Fliplr(0.5), # horizontally flip 50% of all images
#iaa.Flipud(0.2), # vertically flip 20% of all images
#sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
sometimes(
iaa.Affine(
#scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
#translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
#rotate=(-5, 5), # rotate by -45 to +45 degrees
#shear=(-5, 5), # shear by -16 to +16 degrees
#order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
#cval=(0, 255), # if mode is constant, use a cval between 0 and 255
#mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 5),
[
#sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0)
), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(
k=(2, 7)
), # blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(
k=(3, 11)
), # blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
#iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges
#sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
#])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255),
per_channel=0.5), # add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
#iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
#iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add(
(-10, 10), per_channel=0.5
), # change brightness of images (by -10 to 10 of original value)
iaa.Multiply(
(0.5, 1.5), per_channel=0.5
), # change brightness of images (50-150% of original value)
iaa.ContrastNormalization(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
#iaa.Grayscale(alpha=(0.0, 1.0)),
#sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
#sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
],
random_order=True)
],
random_order=True)
if shuffle: np.random.shuffle(self.images)
def augmentor(self, images):
'Apply data augmentation'
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential(
[
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
sometimes(
iaa.Affine(
scale={
"x": (0.9, 1.1),
"y": (0.9, 1.1)
},
# scale images to 80-120% of their size, individually per axis
translate_percent={
"x": (-0.1, 0.1),
"y": (-0.1, 0.1)
},
# translate by -20 to +20 percent (per axis)
rotate=(-10, 10), # rotate by -45 to +45 degrees
shear=(-5, 5), # shear by -16 to +16 degrees
order=[0, 1],
# use nearest neighbour or bilinear interpolation (fast)
cval=(
0, 255
), # if mode is constant, use a cval between 0 and 255
mode=ia.ALL
# use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 5),
[
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
# convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur((0, 1.0)),
# blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(3, 5)),
# blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 5)),
# blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)),
# sharpen images
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.01 * 255), per_channel=0.5),
# add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.05), per_channel=0.5),
# randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.01, 0.03),
size_percent=(0.01, 0.02),
per_channel=0.2),
]),
iaa.Invert(0.01, per_channel=True),
# invert color channels
iaa.Add((-2, 2), per_channel=0.5),
# change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation((-1, 1)),
# change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.9, 1.1), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-1, 0),
first=iaa.Multiply(
(0.9, 1.1), per_channel=True),
second=iaa.ContrastNormalization((0.9, 1.1)))
]),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5),
sigma=0.25)),
# move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
# sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
],
random_order=True)
return seq.augment_images(images)
def __call__(self, sample):
image, pose = sample['image'], sample['pose'].reshape([-1, 2])
sometimes = lambda aug: iaa.Sometimes(0.3, aug)
seq = iaa.Sequential(
[
# Apply the following augmenters to most images.
sometimes(
iaa.CropAndPad(percent=(-0.25, 0.25),
pad_mode=["edge"],
keep_size=False)),
sometimes(
iaa.Affine(scale={
"x": (0.75, 1.25),
"y": (0.75, 1.25)
},
translate_percent={
"x": (-0.25, 0.25),
"y": (-0.25, 0.25)
},
rotate=(-45, 45),
shear=(-5, 5),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL)),
iaa.SomeOf(
(0, 3),
[
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
# iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
iaa.MotionBlur(k=5, angle=[-45, 45])
]),
iaa.OneOf([
iaa.AdditiveGaussianNoise(loc=0,
scale=(0.0, 0.05 * 255),
per_channel=0.5),
iaa.AdditivePoissonNoise(lam=(0, 8),
per_channel=True),
]),
iaa.OneOf([
iaa.Add((-10, 10), per_channel=0.5),
iaa.Multiply((0.2, 1.2), per_channel=0.5),
iaa.ContrastNormalization(
(0.5, 2.0), per_channel=0.5),
]),
],
# do all of the above augmentations in random order
random_order=True)
],
# do all of the above augmentations in random order
random_order=True)
# augmentation choices
seq_det = seq.to_deterministic()
image_aug = seq_det.augment_images([image])[0]
keypoints_aug = seq_det.augment_keypoints(
[self.pose2keypoints(image, pose)])[0]
return {'image': image_aug, 'pose': self.keypoints2pose(keypoints_aug)}
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = BalloonDataset()
dataset_train.load_balloon(args.dataset, "train")
dataset_train.prepare()
# Validation dataset
dataset_val = BalloonDataset()
dataset_val.load_balloon(args.dataset, "val")
dataset_val.prepare()
# Image augmentation
# http://imgaug.readthedocs.io/en/latest/source/augmenters.html
augmentation = iaa.SomeOf(
(1, 5),
[
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
iaa.Affine(rotate=90), # new to force more horizontal lines
iaa.Affine(
rotate=(-90, 90), mode="edge"
), # mode= "edge" ads straight lines in created empty space
#for this do one of
iaa.OneOf([
iaa.CropAndPad(percent=(-0.3, 0.05),
sample_independently=False,
pad_mode="edge"),
iaa.Crop(percent=(0, 0.05))
]),
iaa.GaussianBlur(sigma=(0.0, 1.0)),
iaa.Multiply((0.5, 1.2)), #changeing brightness
iaa.Grayscale(alpha=(0.0, 0.9))
])
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
print("Training network heads")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=15,
augmentation=augmentation,
layers='heads') # 'heads' or 'all'
print("Training 4+")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=60,
augmentation=augmentation,
layers='4+') # 'heads' or 'all'
print("Train all")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=200,
augmentation=augmentation,
layers='all') # 'heads' or 'all'
print("Train all lower learning rate")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE / 10,
epochs=400,
augmentation=augmentation,
layers='all') # 'heads' or 'all'
# sometimes(iaa.OneOf([
# iaa.EdgeDetect(alpha=(0, 0.7)),
# iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
# ])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# add gaussian noise to images
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
# iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
]),
# iaa.Invert(0.05, per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5),
# change brightness of images (by -10 to 10 of original value)
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# change brightness of images (50-150% of original value)
iaa.ContrastNormalization(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
# iaa.Grayscale(alpha=(0.0, 1.0)),
# sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
# sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
],
random_order=True)
],
random_order=True)
def aug_image(image, annos, jitter):
orig_h, orig_w = image.shape[:2]
