def _augment(self, x, y):
for idx, (xx, yy) in enumerate(zip(x, y)):
if self.hflip and np.random.rand() > 0.5:
x[idx, ...] = kp.flip_axis(xx, axis=1)
y[idx, ...] = kp.flip_axis(yy, axis=1)
if self.vflip and np.random.rand() > 0.5:
x[idx, ...] = kp.flip_axis(xx, axis=0)
y[idx, ...] = kp.flip_axis(yy, axis=0)
if self.brg is not None:
# random_brightness per channel
for chn in range(xx.shape[-1]):
u = np.random.uniform(*self.brg)
x[idx, ..., chn:chn + 1] = kp.apply_brightness_shift(
xx[..., chn:chn + 1], u)
if self.rg > 0 or self.wrg > 0 or self.hrg > 0 or self.zrg is not None:
params = {
'theta': 0,
'tx': 0,
'ty': 0,
'shear': 0,
'zx': 1,
'zy': 1
}
if self.rg > 0:
params['theta'] = np.random.uniform(-self.rg, self.rg)
if self.wrg > 0 or self.hrg > 0:
h, w = xx.shape[0], xx.shape[1]
params['tx'] = np.random.uniform(-self.wrg, self.wrg) * w
params['ty'] = np.random.uniform(-self.hrg, self.hrg) * h
if self.zrg is not None:
params['zx'] = np.random.uniform(*self.zrg, 1)[0]
params['zy'] = params['zx']
x[idx, ...] = kp.apply_affine_transform(xx, **params)
y[idx, ...] = kp.apply_affine_transform(yy, **params)
return x, y
def random_flip(img, mask, u=0.5):
if np.random.random() < u:
img = image.flip_axis(img, 1)
mask = image.flip_axis(mask, 1)
if np.random.random() < u:
img = image.flip_axis(img, 0)
mask = image.flip_axis(mask, 0)
return img, mask
def apply_transform(self, x, transform_parameters):
"""Applies a transformation to an image according to given parameters.
# Arguments
x: 3D tensor, single image.
transform_parameters: Dictionary with string - parameter pairs
describing the transformation.
Currently, the following parameters
from the dictionary are used:
- `'theta'`: Float. Rotation angle in degrees.
- `'tx'`: Float. Shift in the x direction.
- `'ty'`: Float. Shift in the y direction.
- `'shear'`: Float. Shear angle in degrees.
- `'zx'`: Float. Zoom in the x direction.
- `'zy'`: Float. Zoom in the y direction.
- `'flip_horizontal'`: Boolean. Horizontal flip.
- `'flip_vertical'`: Boolean. Vertical flip.
- `'channel_shift_intencity'`: Float. Channel shift intensity.
- `'brightness'`: Float. Brightness shift intensity.
# Returns
A transformed version of the input (same shape).
"""
# x is a single image, so it doesn't have image number at index 0
img_row_axis = self.row_axis - 1
img_col_axis = self.col_axis - 1
img_channel_axis = self.channel_axis - 1
x = apply_affine_transform(x,
transform_parameters.get('theta', 0),
transform_parameters.get('tx', 0),
transform_parameters.get('ty', 0),
transform_parameters.get('shear', 0),
transform_parameters.get('zx', 1),
transform_parameters.get('zy', 1),
row_axis=img_row_axis,
col_axis=img_col_axis,
channel_axis=img_channel_axis,
fill_mode=self.fill_mode,
cval=self.cval,
order=self.interpolation_order)
if transform_parameters.get('channel_shift_intensity') is not None:
x = apply_channel_shift(
x, transform_parameters['channel_shift_intensity'],
img_channel_axis)
if transform_parameters.get('flip_horizontal', False):
x = flip_axis(x, img_col_axis)
if transform_parameters.get('flip_vertical', False):
x = flip_axis(x, img_row_axis)
if transform_parameters.get('brightness') is not None:
x = apply_brightness_shift(x, transform_parameters['brightness'])
return x
def img_augment(imgs_p, masks_p, new_size, rotate, zoom, h_flip, v_flip, crop,
batch_size, val):
# print(imgs_p)
imgs = [cv2.imread(p, 0)[1:, 1:] for p in imgs_p]
masks = [cv2.imread(p, 0)[1:, 1:] for p in masks_p]
imgs_container = []
masks_container = []
for img, mask in zip(imgs, masks):
# img_o = cv2.merge((img, img, img))
img_o = img[..., np.newaxis]
mask = mask / 255.0
mask_o = mask[..., np.newaxis]
imgs_container.append(img_o)
masks_container.append(mask_o)
img_r, mask_r = random_rotation(img_o, mask_o, rotate)
imgs_container.append(img_r)
masks_container.append(mask_r)
img_z, mask_z = random_zoom(img_o, mask_o, zoom)
imgs_container.append(img_z)
masks_container.append(mask_z)
if h_flip:
img_f = flip_axis(img_o, 1)
mask_f = flip_axis(mask_o, 1)
imgs_container.append(img_f)
masks_container.append(mask_f)
if v_flip:
img_f = flip_axis(img_o, 0)
mask_f = flip_axis(mask_o, 0)
imgs_container.append(img_f)
masks_container.append(mask_f)
img_batches, mask_batches = class_mask(
imgs_container,
masks_container,
new_size,
crop)
if batch_size == 0:
batch_size = len(img_batches)
return np.array(img_batches[0:batch_size]), np.array(mask_batches[0:batch_size])
def transform_pic(pic):
theta = np.random.uniform(-20, 20)
ty = np.random.uniform(-0.2, 0.2)
tx = np.random.uniform(-0.2, 0.2)
shear = np.random.uniform(-0.2, 0.2)
zy = np.random.uniform(0.8, 1)
zx = np.random.uniform(0.8, 1)
pic = apply_affine_transform(pic,
theta=theta,
tx=tx,
ty=ty,
shear=shear,
zx=zx,
zy=zy,
channel_axis=2)
if random.randint(0, 1):
pic = flip_axis(pic, 0)
if random.randint(0, 1):
pic = flip_axis(pic, 1)
return pic
def get_batch():
index = 1
global current_index
B = np.zeros(shape=(batch_size, IMAGE_SIZE, IMAGE_SIZE, 3))
L = np.zeros(shape=(batch_size))
while index < batch_size:
try:
img = load_img(images[current_index],
target_size=(IMAGE_SIZE, IMAGE_SIZE))
img = img_to_array(img)
img /= 255.
# if cnn == 'ResNet50': # imagenet pretrained
# mean = np.array([0.485, 0.456, 0.406])
# std = np.array([0.229, 0.224, 0.225])
# img = (img - mean)/std
## data augmentation
# random width and height shift
img = random_shift(img, 0.2, 0.2)
# random rotation
img = random_rotation(img, 10)
# random horizental flip
flip_horizontal = (np.random.random() < 0.5)
if flip_horizontal:
img = flip_axis(img, axis=1)
# # random vertical flip
# flip_vertical = (np.random.random() < 0.5)
# if flip_vertical:
# img = flip_axis(img, axis=0)
# #cutout
# eraser = get_random_eraser(v_l=0, v_h=1, pixel_level=False)
# img = eraser(img)
B[index] = img
L[index] = labels[current_index]
index = index + 1
current_index = current_index + 1
except:
traceback.print_exc()
# print("Ignore image {}".format(images[current_index]))
current_index = current_index + 1
# B = np.rollaxis(B, 3, 1)
return B, np_utils.to_categorical(L, num_classes)
def random_flip(img, masks, masks2, u=0.5):
if np.random.random() < u:
img = image.flip_axis(img, 1)
for i in range(masks.shape[0]):
masks[i] = image.flip_axis(masks[i], 1)
for i in range(masks2.shape[0]):
masks2[i] = image.flip_axis(masks2[i], 1)
if np.random.random() < u:
img = image.flip_axis(img, 0)
for i in range(masks.shape[0]):
masks[i] = image.flip_axis(masks[i], 0)
for i in range(masks2.shape[0]):
masks2[i] = image.flip_axis(masks2[i], 0)
return img, masks, masks2
def keras(self, img):
return np.ascontiguousarray(keras.flip_axis(img, axis=0))
def augment(self, face_img: np.ndarray, nface_img: np.ndarray, leye_img: np.ndarray, reye_img: np.ndarray,
lndmks: np.ndarray , y: np.ndarray=None, keep: bool=False, last_state: dict=None):
"""
Main function, used to augment a series of images and data. If keep is True, use last_state to define
the augmentations to apply; otherwise, compute new augmentations.
:param face_img: Original (not-normalized) face image
:param nface_img: Normalized face image
:param leye_img: Normalized left eye image
:param reye_img: Normalized right eye image
:param lndmks: 3D Landmarks
:param y: label
:param keep: If true, use last_state to define the augmentations to apply; otherwise, compute new augmentations.
:param last_state: Optional dictionary specifying the last augmentations applied.
:return: augmented images and data, and current_state
"""
# Check if we use existing augmentations or we have to compute new ones
if keep:
assert(set(last_state) == set(self.last_state))
else:
last_state = dict(self.last_state)
# Vertical flip
if self.dict['vertical_flip']:
if (not keep and np.random.random() < self.flip_prob) \
or (keep and last_state['vertical_flip']):
face_img = flip_axis(face_img, 0)
nface_img = flip_axis(nface_img, 0)
leye_img = flip_axis(leye_img, 0)
reye_img = flip_axis(reye_img, 0)
lndmks[:, 1] = - lndmks[:, 1]
y[1] = -y[1]
last_state['vertical_flip'] = True
else:
last_state['vertical_flip'] = False
# Horizontal flip
if self.dict['horizontal_flip']:
if (not keep and np.random.random() < self.flip_prob) \
or (keep and last_state['horizontal_flip']):
face_img = flip_axis(face_img, 1)
nface_img = flip_axis(nface_img, 1)
leye_img_t = flip_axis(leye_img, 1) # change left-right order
reye_img_t = flip_axis(reye_img, 1)
leye_img = reye_img_t
reye_img = leye_img_t
lndmks[:, 0] = - lndmks[:, 0]
y[0] = -y[0]
last_state['horizontal_flip'] = True
else:
last_state['horizontal_flip'] = False
# Rotation
if keep:
theta = last_state['rotation']
else:
if self.dict['rotation_range'] > 0. and np.random.random() < self.rotation_prob:
theta = np.pi / 180 * np.random.uniform(-self.dict['rotation_range'], self.dict['rotation_range'])
else:
theta = 0
last_state['rotation'] = theta
# Translation in y (in pixels)
if keep:
tx = last_state['shift_x']
else:
if self.dict['height_shift_range'] > 0. and np.random.random() < self.shift_prob:
tx = np.random.uniform(-self.dict['height_shift_range'], self.dict['height_shift_range'])
else:
tx = 0
last_state['shift_x'] = tx
# Translation in x (in pixels)
if keep:
ty = last_state['shift_y']
else:
if self.dict['width_shift_range'] > 0. and np.random.random() < self.shift_prob:
ty = np.random.uniform(-self.dict['width_shift_range'], self.dict['width_shift_range'])
else:
ty = 0
last_state['shift_y'] = ty
# Zoom
if keep:
z = last_state['zoom']
else:
if self.dict['zoom_range'][0] != 1 and self.dict['zoom_range'][1] != 1 and \
np.random.random() < self.zoom_prob:
z = np.random.uniform(self.dict['zoom_range'][0], self.dict['zoom_range'][1])
else:
z = 1
last_state['zoom'] = z
# Apply composition of transformations
transform_matrix = None
if theta != 0:
rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
[np.sin(theta), np.cos(theta), 0],
[0, 0, 1]])
transform_matrix = rotation_matrix
if tx != 0 or ty != 0:
shift_matrix = np.array([[1, 0, tx],
[0, 1, ty],
[0, 0, 1]])
transform_matrix = shift_matrix if transform_matrix is None else np.dot(transform_matrix, shift_matrix)
if z != 1:
zoom_matrix = np.array([[z, 0, 0],
[0, z, 0],
[0, 0, 1]])
transform_matrix = zoom_matrix if transform_matrix is None else np.dot(transform_matrix, zoom_matrix)
if transform_matrix is not None:
face_img = apply_transform_matrix(self, face_img, transform_matrix)
nface_img = apply_transform_matrix(self, nface_img, transform_matrix)
leye_img = apply_transform_matrix(self, leye_img, transform_matrix)
reye_img = apply_transform_matrix(self, reye_img, transform_matrix)
# Illumination
if self.dict['illumination_range'][0] > 0. and self.dict['illumination_range'][0] != 1 \
and self.dict['illumination_range'][1] != 1:
if not keep and np.random.random() < self.illumination_prob:
[face_img, nface_img, leye_img, reye_img], last_state['illumination'] = \
modify_illumination([face_img, nface_img, leye_img, reye_img], self.dict['illumination_range'])
elif keep and last_state['illumination'] != 1:
[face_img, nface_img, leye_img, reye_img], last_state['illumination'] = \
modify_illumination([face_img, nface_img, leye_img, reye_img], self.dict['illumination_range'],
last_state['illumination'])
# Additive gaussian noise
if self.dict['gaussian_noise_range'][1] > 0.:
if not keep and np.random.random() < self.gaussian_noise_prob:
[face_img, nface_img, leye_img, reye_img], last_state['gauss_var'], last_state['gauss_noise'] = \
add_gaussian_noise([face_img, nface_img, leye_img, reye_img], self.dict['gaussian_noise_range'])
elif keep and last_state['gauss_noise'] != []:
[face_img, nface_img, leye_img, reye_img], last_state['gauss_var'], last_state['gauss_noise'] = \
add_gaussian_noise([face_img, nface_img, leye_img, reye_img], self.dict['gaussian_noise_range'],
last_state['gauss_var'], last_state['gauss_noise'])
return [face_img, nface_img, leye_img, reye_img, lndmks], y, last_state
def keras(self, img):
return np.ascontiguousarray(keras.flip_axis(img, axis=0))
def flip_vertical(x, value, row_axis=1):
if value < 0.5:
return flip_axis(x, row_axis)
else:
return x
def flip_horizontal(x, value, col_axis=2):
if value < 0.5:
return flip_axis(x, col_axis)
else:
return x
def random_transform_extension(self, x, y, seed=None):
"""Randomly augment a single image tensor.
# Arguments
x: 3D tensor, single image.
seed: random seed.
# Returns
A randomly transformed version of the input (same shape).
"""
# x is a single image, so it doesn't have image number at index 0
img_row_axis = self.row_axis - 1
img_col_axis = self.col_axis - 1
img_channel_axis = self.channel_axis - 1
if seed is not None:
np.random.seed(seed)
# use composition of homographies
# to generate final transform that needs to be applied
if self.rotation_range:
theta = np.pi / 180 * np.random.uniform(-self.rotation_range,
self.rotation_range)
else:
theta = 0
if self.height_shift_range:
tx = np.random.uniform(
-self.height_shift_range,
self.height_shift_range) * x.shape[img_row_axis]
else:
tx = 0
if self.width_shift_range:
ty = np.random.uniform(
-self.width_shift_range,
self.width_shift_range) * x.shape[img_col_axis]
else:
ty = 0
if self.shear_range:
shear = np.random.uniform(-self.shear_range, self.shear_range)
else:
shear = 0
if self.zoom_range[0] == 1 and self.zoom_range[1] == 1:
zx, zy = 1, 1
else:
zx, zy = np.random.uniform(self.zoom_range[0], self.zoom_range[1],
2)
transform_matrix = None
if theta != 0:
rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
[np.sin(theta),
np.cos(theta), 0], [0, 0, 1]])
transform_matrix = rotation_matrix
if tx != 0 or ty != 0:
shift_matrix = np.array([[1, 0, tx], [0, 1, ty], [0, 0, 1]])
transform_matrix = shift_matrix if transform_matrix is None else np.dot(
transform_matrix, shift_matrix)
if shear != 0:
shear_matrix = np.array([[1, -np.sin(shear), 0],
[0, np.cos(shear), 0], [0, 0, 1]])
transform_matrix = shear_matrix if transform_matrix is None else np.dot(
transform_matrix, shear_matrix)
if zx != 1 or zy != 1:
zoom_matrix = np.array([[zx, 0, 0], [0, zy, 0], [0, 0, 1]])
transform_matrix = zoom_matrix if transform_matrix is None else np.dot(
transform_matrix, zoom_matrix)
if transform_matrix is not None:
hx, wx = x.shape[img_row_axis], x.shape[img_col_axis]
hy, wy = y.shape[img_row_axis], y.shape[img_col_axis]
transform_matrix_x = kimage.transform_matrix_offset_center(
transform_matrix, hx, wx)
transform_matrix_y = kimage.transform_matrix_offset_center(
transform_matrix, hy, wy)
x = apply_transform(x,
transform_matrix_x,
img_channel_axis,
fill_mode=self.fill_mode,
cval=self.cval)
y = apply_transform(y,
transform_matrix_y,
img_channel_axis,
fill_mode=self.fill_mode,
cval=self.cval)
if self.channel_shift_range != 0:
x = image.random_channel_shift(x, self.channel_shift_range,
img_channel_axis)
if self.horizontal_flip:
if np.random.random() < 0.5:
x = kimage.flip_axis(x, img_col_axis)
y = kimage.flip_axis(y, img_col_axis)
if self.vertical_flip:
if np.random.random() < 0.5:
x = kimage.flip_axis(x, img_row_axis)
y = kimage.flip_axis(y, img_row_axis)
return x, y
int(bbox[0] * w):int(bbox[2] * w) - 1, 0] = 100
x = apply_affine_transform(
x,
transform_parameters.get('theta', 0),
transform_parameters.get('tx', 0),
transform_parameters.get('ty', 0),
transform_parameters.get('shear', 0),
transform_parameters.get('zx', 1),
transform_parameters.get('zy', 1),
row_axis=0,
col_axis=1,
channel_axis=2,
fill_mode='nearest',
cval=0.)
if transform_parameters.get('flip_horizontal', False):
x = flip_axis(x, 1)
if transform_parameters.get('flip_vertical', False):
x = flip_axis(x, 0)
x = x[:, :, 0]
arr_h = np.max(x, axis=1)
arr_w = np.max(x, axis=0)
i = 0
while (i < h and arr_h[i] < 1e-14):
i += 1
y1 = i
i = h - 1
while (i >= 0 and arr_h[i] < 1e-14):
i -= 1
y2 = i
i = 0
while (i < w and arr_w[i] < 1e-14):
