def img2xy(self, img_path, gal, gpr, augment=0):
'''
Single img file to xys for all blocks and channels
returns:
xs:
cropped window, grayscale[0~255]=>[0~1]
shape ( #blocks, 1, win_shape[0], win_shape[1] )
ys:
gaussian heatmaps of same w, h as xs, one channel for one kpt
idx:
(img_path, block) for one x sample
'''
idxs, imgs, df = super().img2x(img_path, gal,
gpr) # imgs: (N, 1, h, w)
h, w = imgs.shape[2], imgs.shape[3]
gpr = Gpr(gpr) if type(gpr) == str else gpr
xs, ys, coords = [], [], []
for (img_path, b), img, block_df in zip(idxs, imgs, df):
nrows = gal.header[f'Block{b}'][Gal.N_ROWS]
ncols = gal.header[f'Block{b}'][Gal.N_COLS]
kpts = KeypointsOnImage([
Keypoint(x=block_df.loc[1, 1]['X'], y=block_df.loc[1, 1]['Y']),
Keypoint(x=block_df.loc[1, ncols]['X'],
y=block_df.loc[1, ncols]['Y']),
Keypoint(x=block_df.loc[nrows, ncols]['X'],
y=block_df.loc[nrows, ncols]['Y']),
Keypoint(x=block_df.loc[nrows, 1]['X'],
y=block_df.loc[nrows, 1]['Y'])
],
shape=(h, w))
if augment <= 0:
xs.append(img)
coord = self.to_Lcoord(kpts.to_xy_array())
ys.append(
self.coord2heatmap(coord, (img.shape[-2], img.shape[-1])))
coords.append(coord.flatten())
else:
for i in range(augment):
img_aug, kpts_aug = self.aug_seq(
image=(img[0] * 255).astype('uint8'),
keypoints=kpts) # img: (1, w, h) -> (w, h)
coord = self.to_Lcoord(kpts_aug.to_xy_array()) # (3, 2)
if self.check_out_of_bounds(img_aug.shape, coord):
continue # skip if coord out of bounds
xs.append(np.array([img_aug / 255]))
ys.append(
self.coord2heatmap(
coord, (img_aug.shape[-2], img_aug.shape[-1])))
coords.append(coord.flatten())
return np.stack(xs), np.stack(ys), np.stack(coords)
def genData(num=24, shift=10, isTrain=True, deleteOldFile=True):
"""
augument picture and landmarks, output a txt file
:param num: number of augumented picture
:param shift: make your picture little bigger than original facebox
:param isTrain: choose different path of pic and landmarks
:param deleteOldFile: delete old txt file
:return:
"""
if osp.exists("data/landmark.txt"):
if deleteOldFile:
os.remove("data/landmark.txt")
else:
print("WARNING: continue to write on landmark.txt")
if isTrain:
data_landmarks = np.loadtxt(
train_landmarks_path,
usecols=([i for i in range(NUM_LANDMARKS * 2)]),
dtype=np.float)
data_faceArea = np.loadtxt(
train_landmarks_path,
usecols=([NUM_LANDMARKS * 2 + i for i in range(4)]),
dtype=np.float)
data_image = np.loadtxt(train_landmarks_path,
usecols=(-1),
dtype=np.str)
else:
data_landmarks = np.loadtxt(
test_landmarks_path,
usecols=([i for i in range(NUM_LANDMARKS * 2)]),
dtype=np.float)
data_faceArea = np.loadtxt(
test_landmarks_path,
usecols=([NUM_LANDMARKS * 2 + i for i in range(4)]),
dtype=np.float)
data_image = np.loadtxt(test_landmarks_path,
usecols=(-1),
dtype=np.str)
# https://nbviewer.jupyter.org/github/aleju/imgaug-doc/blob/master/notebooks/B01%20-%20Augment%20Keypoints.ipynb
for _i in range(len(data_image)):
IND = _i
sometimes = lambda aug: iaa.Sometimes(0.4, aug)
sometimes_01 = lambda aug: iaa.Sometimes(0.18, aug)
# load pic, add a new dim and stack 20 of it together
image_path = osp.join(img_path, data_image[IND])
image = cv2.imread(image_path)
cols = data_faceArea[IND][
0] - shift if data_faceArea[IND][0] - shift > 0 else 0
rows = data_faceArea[IND][
1] - shift if data_faceArea[IND][1] - shift > 0 else 0
weight = data_faceArea[IND][2] + shift if data_faceArea[IND][
2] + shift < image.shape[1] else image.shape[1]
height = data_faceArea[IND][3] + shift if data_faceArea[IND][
3] + shift < image.shape[0] else image.shape[0]
image = image[int(rows):int(height), int(cols):int(weight), :]
# images = np.concatenate((
# [np.expand_dims(image, axis=0)] * 20
# ), dtype=np.uint8)
# landmarks
kpsoi = KeypointsOnImage([
Keypoint(x=data_landmarks[IND][i] - cols,
y=data_landmarks[IND][i + 1] - rows)
for i in range(0, NUM_LANDMARKS * 2, 2)
],
shape=image.shape)
# kpsois = [kpsoi.to_xy_array()]*20
seq = iaa.Sequential([
iaa.Fliplr(p=0.35),
sometimes(
iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode=ia.ALL,
pad_cval=(0, 255))),
sometimes(
iaa.Affine(
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
}, # scale images to 80-120% of their size, individually per axis
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
}, # translate by -20 to +20 percent (per axis)
rotate=(-15, 15), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[
0,
1
], # use nearest neighbour or bilinear interpolation (fast)
)),
iaa.AddToHueAndSaturation((-25, 25)),
iaa.OneOf([
iaa.Multiply((0.5, 1.5)),
iaa.FrequencyNoiseAlpha(exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5)),
second=iaa.ContrastNormalization(
(0.5, 2.0)))
]),
sometimes_01(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
sometimes_01(iaa.PerspectiveTransform(scale=(0.01, 0.1))),
])
df = pd.DataFrame(kpsoi.to_xy_array().reshape(-1)).T
df.insert(
0, 'path',
osp.join(train_path_for_save,
data_image[IND][data_image[IND].rfind("/") + 1:]))
for index in range(num):
image_aug, kpsoi_aug = seq(image=image, keypoints=kpsoi)
ld = kpsoi_aug.to_xy_array().reshape(-1)
if check_ld_boundary(ld, image_aug.shape):
continue
_path = osp.join(
train_path_for_save,
str(index) + "_" +
data_image[IND][data_image[IND].rfind("/") + 1:])
# cv2.imshow(
# "image",
# np.hstack([
# kpsoi.draw_on_image(image, size=7),
# kpsoi_aug.draw_on_image(image_aug, size=7)
# ])
# )
# Check our landmarks
# for ind in range(0, NUM_LANDMARKS*2, 2):
# cv2.circle(image_aug, (ld[ind], ld[ind+1]), 1, (76, 201, 255), 1)
# cv2.imshow("img", image_aug)
# cv2.waitKey(0)
df2 = pd.DataFrame(ld).T
df2.insert(0, 'path', _path)
df = pd.concat([df, df2])
cv2.imwrite(_path, image_aug)
df.to_csv(landmark_path_for_save,
sep=' ',
header=None,
index=None,
mode='a')
cv2.imwrite(
osp.join(train_path_for_save,
data_image[IND][data_image[IND].rfind("/") + 1:]),
image_aug)
def img2xy(self,
ds_name,
img_path,
gal_path,
gpr_path,
row_inc,
col_inc,
scale,
h_flip,
v_flip,
blur,
noise,
augment=0,
cut_pp=None):
if ds_name in self.gal_cache:
gal = self.gal_cache[ds_name]
else:
gal = make_fake_gal(
Gpr(gpr_path)) if gal_path == '' else Gal(gal_path)
self.gal_cache[ds_name] = gal
if img_path in self.img_cache:
idxs, imgs, df, scales = self.img_cache[img_path]
else:
idxs, imgs, df, scales = self.img2x(
img_path, gal, Gpr(gpr_path)) # imgs: (N, 1, h, w)
self.img_cache[img_path] = idxs, imgs, df, scales
if (imgs is None) or (df is None):
print(f'{img_path} failed.')
return None, None, None, None
h, w = imgs.shape[-2], imgs.shape[-1]
xs, ys, isori = [], [], []
for (img_path, b, c), img, block_df in zip(idxs, imgs, df):
nrows = gal.header[f'Block{b}'][Gal.N_ROWS]
ncols = gal.header[f'Block{b}'][Gal.N_COLS]
kpts = KeypointsOnImage([
Keypoint(x=block_df.loc[1, 1]['X'], y=block_df.loc[1, 1]['Y']),
Keypoint(x=block_df.loc[nrows, 1]['X'],
y=block_df.loc[nrows, 1]['Y']),
Keypoint(x=block_df.loc[nrows, ncols]['X'],
y=block_df.loc[nrows, ncols]['Y']),
Keypoint(x=block_df.loc[1, ncols]['X'],
y=block_df.loc[1, ncols]['Y']),
],
shape=(h, w))
# bsa
bsa = BlockSizeAugmentor((nrows, ncols), cut_pp)
img_ = []
kpts_ = kpts.copy()
for im in img:
im, kpts, _ = bsa.augment(im, kpts_.to_xy_array(), row_inc,
col_inc)
img_.append(im)
img = np.stack(img_).astype('uint8')
coord = self.to_Lcoord(kpts.to_xy_array())
if self.check_out_of_bounds(img.shape[1:], coord):
continue
img = np.moveaxis(img, 0, -1) # (c, h, w) -> (h, w, c)
# h_flip
if h_flip:
aug_func = aug.HorizontalFlip(1)
img, kpts = aug_func(image=img, keypoints=kpts)
# v_flip
if v_flip:
aug_func = aug.VerticalFlip(1)
img, kpts = aug_func(image=img, keypoints=kpts)
# scale
if scale != 1:
aug_func = aug.Affine(scale=scale, mode='wrap')
img, kpts = aug_func(image=img, keypoints=kpts)
if self.check_out_of_bounds(img.shape[:2], kpts.to_xy_array()):
continue
# blur
if blur != 0:
aug_func = aug.GaussianBlur(sigma=blur)
img, kpts = aug_func(image=img, keypoints=kpts)
# noise
if noise != 0:
aug_func = aug.AdditiveGaussianNoise(scale=noise)
img, kpts = aug_func(image=img, keypoints=kpts)
img = np.moveaxis(img, -1, 0) # (h, w, c) -> (c, h, w)
coord = self.to_Lcoord(kpts.to_xy_array())
if self.check_out_of_bounds(img.shape[1:], coord):
continue
xs.append(img)
ys.append(coord.flatten())
isori.append(True)
# augment
if augment >= 1:
augxs, augys, fail_count = [], [], 0
while len(augxs) < augment:
img_aug, coord = self.run_aug(img, kpts)
if fail_count > 5:
return None, None, None, None
if img_aug is None:
fail_count += 1
continue
augxs.append(img_aug)
augys.append(coord.flatten())
xs.extend(augxs)
ys.extend(augys)
isori.extend([False] * len(augxs))
elif np.random.rand() < augment:
img_aug, coord = self.run_aug(img, kpts)
if img_aug is not None:
xs.append(img_aug)
ys.append(coord.flatten())
isori.append(False)
if len(xs) <= 0:
return None, None, None, None
return np.stack(xs) / 255, np.stack(ys), np.stack(isori), (
bsa.row_cut_pp, bsa.col_cut_pp)
def img2xy(self,
img_path,
gal_path,
gpr_path,
augment=0,
bsa_args: dict = None,
blocks=None,
keep_ori=True):
'''
Single img file to xys for all blocks and channels
args:
blocks:
If None, get all blocks in img.
If given list / set, get only the blocks in the list
ys:
top left, bottom left, bottom right XYs of a block (L shape)
(relative to window coords)
shape ( #blocks*2, 6 )
is_ori (1D array):
True : sample is original data
False: sample is augmented data
'''
gal = make_fake_gal(Gpr(gpr_path)) if gal_path == '' else Gal(gal_path)
idxs, imgs, df, scales = super().img2x(
img_path, gal, Gpr(gpr_path)) # imgs: (N, 1, h, w)
if (imgs is None) or (df is None):
return None, None
h, w = imgs.shape[2], imgs.shape[3]
xs, ys, is_ori = [], [], []
for (img_path, b, c), img, block_df in zip(idxs, imgs, df):
if (blocks is not None) and not (b in blocks):
continue
nrows = gal.header[f'Block{b}'][Gal.N_ROWS]
ncols = gal.header[f'Block{b}'][Gal.N_COLS]
kpts = KeypointsOnImage([
Keypoint(x=block_df.loc[1, 1]['X'], y=block_df.loc[1, 1]['Y']),
Keypoint(x=block_df.loc[nrows, 1]['X'],
y=block_df.loc[nrows, 1]['Y']),
Keypoint(x=block_df.loc[nrows, ncols]['X'],
y=block_df.loc[nrows, ncols]['Y']),
Keypoint(x=block_df.loc[1, ncols]['X'],
y=block_df.loc[1, ncols]['Y']),
],
shape=(h, w))
if keep_ori:
xs.append(img)
coord = self.to_Lcoord(kpts.to_xy_array())
ys.append(coord.flatten())
is_ori.append(True)
if bsa_args is not None:
augxs, augys = [], []
while len(augxs) < bsa_args['n']:
row_inc = np.random.choice(bsa_args['row_inc_choice'])
col_inc = np.random.choice(bsa_args['col_inc_choice'])
bsa = BlockSizeAugmentor((nrows, ncols))
img_ = []
kpts_ = kpts.copy()
for im in img:
im, kpts, _ = bsa.augment(im, kpts_.to_xy_array(),
row_inc, col_inc)
img_.append(im)
img = np.stack(img_).astype('uint8')
coord = self.to_Lcoord(kpts.to_xy_array())
if self.check_out_of_bounds(img.shape[1:], coord):
continue
augxs.append(img)
augys.append(coord.flatten())
xs.extend(augxs)
ys.extend(augys)
is_ori.extend([False] * len(augxs))
if augment >= 1:
augxs, augys = [], []
while len(augxs) < augment:
img_aug, coord = self.run_aug(img, kpts)
if img_aug is None: continue
augxs.append(img_aug)
augys.append(coord.flatten())
xs.extend(augxs)
ys.extend(augys)
is_ori.extend([False] * len(augxs))
elif np.random.rand() < augment:
img_aug, coord = self.run_aug(img, kpts)
if img_aug is None: continue
xs.append(img_aug)
ys.append(coord.flatten())
is_ori.append(False)
return np.stack(xs) / 255, np.stack(ys), np.array(is_ori)
