def __getitem__(self, index):
if index == 0:
shuffle(self.train_lines)
lines = self.train_lines
#-------------------------------------------------#
# 进行数据增强
#-------------------------------------------------#
img, y = self.get_random_data(lines[index], [self.input_size[0],self.input_size[1]], random=self.is_train)
batch_hm = np.zeros((self.output_size[0], self.output_size[1], self.num_classes), dtype=np.float32)
batch_wh = np.zeros((self.output_size[0], self.output_size[1], 2), dtype=np.float32)
batch_reg = np.zeros((self.output_size[0], self.output_size[1], 2), dtype=np.float32)
batch_reg_mask = np.zeros((self.output_size[0], self.output_size[1]), dtype=np.float32)
if len(y) != 0:
#-------------------------------------------------#
# 转换成相对于特征层的大小
#-------------------------------------------------#
boxes = np.array(y[:,:4],dtype=np.float32)
boxes[:,0] = boxes[:,0] / self.input_size[1] * self.output_size[1]
boxes[:,1] = boxes[:,1] / self.input_size[0] * self.output_size[0]
boxes[:,2] = boxes[:,2] / self.input_size[1] * self.output_size[1]
boxes[:,3] = boxes[:,3] / self.input_size[0] * self.output_size[0]
for i in range(len(y)):
bbox = boxes[i].copy()
bbox = np.array(bbox)
#-------------------------------------------------#
# 防止超出特征层的范围
#-------------------------------------------------#
bbox[[0, 2]] = np.clip(bbox[[0, 2]], 0, self.output_size[1] - 1)
bbox[[1, 3]] = np.clip(bbox[[1, 3]], 0, self.output_size[0] - 1)
cls_id = int(y[i, -1])
h, w = bbox[3] - bbox[1], bbox[2] - bbox[0]
if h > 0 and w > 0:
radius = gaussian_radius((math.ceil(h), math.ceil(w)))
radius = max(0, int(radius))
#-------------------------------------------------#
# 计算真实框所属的特征点
#-------------------------------------------------#
ct = np.array([(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2], dtype=np.float32)
ct_int = ct.astype(np.int32)
#----------------------------#
# 绘制高斯热力图
#----------------------------#
batch_hm[:, :, cls_id] = draw_gaussian(batch_hm[:, :, cls_id], ct_int, radius)
batch_wh[ct_int[1], ct_int[0]] = 1. * w, 1. * h
batch_reg[ct_int[1], ct_int[0]] = ct - ct_int
batch_reg_mask[ct_int[1], ct_int[0]] = 1
img = np.array(img, dtype=np.float32)[:,:,::-1]
img = np.transpose(preprocess_image(img), (2, 0, 1))
return img, batch_hm, batch_wh, batch_reg, batch_reg_mask
def generate_sample_face(image,
landmarks,
detector,
input_resolution=256,
output_resolution=64):
num_landmarks = len(landmarks)
detected_faces = utils.get_face_bbox(image, detector)
outputs = list()
if len(detected_faces) > 0:
for i, rect in enumerate(detected_faces):
center = [(rect.left() + rect.right()) / 2,
(rect.top() + rect.bottom()) / 2]
center[1] = center[1] - (rect.bottom() - rect.top()) * 0.12
# scale = (rect.right() - rect.left() +
# rect.bottom() - rect.top()) / 195.0
scale = 2.0
cropped_image = utils.crop(image,
center,
scale,
resolution=input_resolution)
heatmaps = np.zeros(
(output_resolution, output_resolution, num_landmarks))
transformed_landmarks = []
for j in range(num_landmarks):
ldmk = utils.transform(landmarks[j] + 1,
center,
scale,
resolution=output_resolution)
transformed_landmarks.append(ldmk)
tmp = utils.draw_gaussian(heatmaps[:, :, j], ldmk, 1)
heatmaps[:, :, j] = tmp
outputs.append({
'image': cropped_image / 255,
'heatmaps': heatmaps,
'center': center,
'scale': scale,
'pts': transformed_landmarks
})
return outputs
def __getitem__(self, idx):
img_name = self.img_names[idx]
pil_image = Image.open(img_name)
if pil_image.mode != "RGB":
# if input is grayscale image, convert it to 3 channel image
if self.enhance:
pil_image = power_transform(pil_image, 0.5)
temp_image = Image.new('RGB', pil_image.size)
temp_image.paste(pil_image)
pil_image = temp_image
image = np.array(pil_image)
if self.gray_scale:
image = rgb2gray(image)
image = np.expand_dims(image, axis=2)
image = np.concatenate((image, image, image), axis=2)
image = image * 255.0
image = image.astype(np.uint8)
if not self.detect_face:
center = [450 // 2, 450 // 2 + 0]
if self.center_shift != 0:
center[0] += int(
np.random.uniform(-self.center_shift, self.center_shift))
center[1] += int(
np.random.uniform(-self.center_shift, self.center_shift))
scale = 1.8
else:
detected_faces = self.face_detector.detect_image(image)
if len(detected_faces) > 0:
box = detected_faces[0]
left, top, right, bottom, _ = box
center = [
right - (right - left) / 2.0, bottom - (bottom - top) / 2.0
]
center[1] = center[1] - (bottom - top) * 0.12
scale = (right - left + bottom - top) / 195.0
else:
center = [450 // 2, 450 // 2 + 0]
scale = 1.8
if self.center_shift != 0:
shift = self.center * self.center_shift / 450
center[0] += int(np.random.uniform(-shift, shift))
center[1] += int(np.random.uniform(-shift, shift))
base_name = os.path.basename(img_name)
landmarks_base_name = base_name[:-4] + '_pts.mat'
landmarks_name = os.path.join(self.landmarks_dir, landmarks_base_name)
if os.path.isfile(landmarks_name):
mat_data = sio.loadmat(landmarks_name)
landmarks = mat_data['pts_2d']
elif os.path.isfile(landmarks_name[:-8] + '.pts.npy'):
landmarks = np.load(landmarks_name[:-8] + '.pts.npy')
else:
landmarks = []
heatmap = []
if landmarks != []:
new_image, new_landmarks = cv_crop(image, landmarks, center, scale,
256, self.center_shift)
tries = 0
while self.center_shift != 0 and tries < 5 and (
np.max(new_landmarks) > 240 or np.min(new_landmarks) < 15):
center = [450 // 2, 450 // 2 + 0]
scale += 0.05
center[0] += int(
np.random.uniform(-self.center_shift, self.center_shift))
center[1] += int(
np.random.uniform(-self.center_shift, self.center_shift))
new_image, new_landmarks = cv_crop(image, landmarks, center,
scale, 256,
self.center_shift)
tries += 1
if np.max(new_landmarks) > 250 or np.min(new_landmarks) < 5:
center = [450 // 2, 450 // 2 + 0]
scale = 2.25
new_image, new_landmarks = cv_crop(image, landmarks, center,
scale, 256, 100)
assert (np.min(new_landmarks) > 0 and np.max(new_landmarks) < 256), \
"Landmarks out of boundary!"
image = new_image
landmarks = new_landmarks
heatmap = np.zeros((self.num_lanmdkars, 64, 64))
for i in range(self.num_lanmdkars):
if landmarks[i][0] > 0:
heatmap[i] = draw_gaussian(heatmap[i],
landmarks[i] / 4.0 + 1, 1)
sample = {'image': image, 'heatmap': heatmap, 'landmarks': landmarks}
if self.transform:
sample = self.transform(sample)
return sample
def generate(self, train=True):
while True:
if train:
# 打乱
shuffle(self.train_lines)
lines = self.train_lines
else:
shuffle(self.val_lines)
lines = self.val_lines
batch_images = np.zeros((self.batch_size, self.input_size[0],
self.input_size[1], self.input_size[2]),
dtype=np.float32)
batch_hms = np.zeros((self.batch_size, self.output_size[0],
self.output_size[1], self.num_classes),
dtype=np.float32)
batch_whs = np.zeros((self.batch_size, self.max_objects, 2),
dtype=np.float32)
batch_regs = np.zeros((self.batch_size, self.max_objects, 2),
dtype=np.float32)
batch_reg_masks = np.zeros((self.batch_size, self.max_objects),
dtype=np.float32)
batch_indices = np.zeros((self.batch_size, self.max_objects),
dtype=np.float32)
b = 0
for annotation_line in lines:
img, y = self.get_random_data(annotation_line,
self.input_size[0:2],
random=train)
if len(y) != 0:
boxes = np.array(y[:, :4], dtype=np.float32)
boxes[:, 0] = boxes[:, 0] / self.input_size[
1] * self.output_size[1]
boxes[:, 1] = boxes[:, 1] / self.input_size[
0] * self.output_size[0]
boxes[:, 2] = boxes[:, 2] / self.input_size[
1] * self.output_size[1]
boxes[:, 3] = boxes[:, 3] / self.input_size[
0] * self.output_size[0]
for i in range(len(y)):
bbox = boxes[i].copy()
bbox = np.array(bbox)
bbox[[0, 2]] = np.clip(bbox[[0, 2]], 0,
self.output_size[1] - 1)
bbox[[1, 3]] = np.clip(bbox[[1, 3]], 0,
self.output_size[0] - 1)
cls_id = int(y[i, -1])
h, w = bbox[3] - bbox[1], bbox[2] - bbox[0]
if h > 0 and w > 0:
ct = np.array([(bbox[0] + bbox[2]) / 2,
(bbox[1] + bbox[3]) / 2],
dtype=np.float32)
ct_int = ct.astype(np.int32)
# 获得热力图
radius = gaussian_radius((math.ceil(h), math.ceil(w)))
radius = max(0, int(radius))
batch_hms[b, :, :, cls_id] = draw_gaussian(
batch_hms[b, :, :, cls_id], ct_int, radius)
batch_whs[b, i] = 1. * w, 1. * h
# 计算中心偏移量
batch_regs[b, i] = ct - ct_int
# 将对应的mask设置为1，用于排除多余的0
batch_reg_masks[b, i] = 1
# 表示第ct_int[1]行的第ct_int[0]个。
batch_indices[
b, i] = ct_int[1] * self.output_size[0] + ct_int[0]
# 将RGB转化成BGR
img = np.array(img, dtype=np.float32)[:, :, ::-1]
batch_images[b] = preprocess_image(img)
b = b + 1
if b == self.batch_size:
b = 0
yield [
batch_images, batch_hms, batch_whs, batch_regs,
batch_reg_masks, batch_indices
], np.zeros((self.batch_size, ))
batch_images = np.zeros(
(self.batch_size, self.input_size[0],
self.input_size[1], 3),
dtype=np.float32)
batch_hms = np.zeros(
(self.batch_size, self.output_size[0],
self.output_size[1], self.num_classes),
dtype=np.float32)
batch_whs = np.zeros(
(self.batch_size, self.max_objects, 2),
dtype=np.float32)
batch_regs = np.zeros(
(self.batch_size, self.max_objects, 2),
dtype=np.float32)
batch_reg_masks = np.zeros(
(self.batch_size, self.max_objects), dtype=np.float32)
batch_indices = np.zeros(
(self.batch_size, self.max_objects), dtype=np.float32)
def __getitem__(self, idx):
imgpath = os.path.join(self.img_root,
self.landmarks_frame.iloc[idx, 0])
pil_image = Image.open(imgpath)
if pil_image.mode != "RGB":
# if input is grayscale image, convert it to 3 channel image
if self.enhance:
pil_image = power_transform(pil_image, 0.5)
temp_image = Image.new('RGB', pil_image.size)
temp_image.paste(pil_image)
pil_image = temp_image
image = np.array(pil_image)
if self.gray_scale:
image = rgb2gray(image)
image = np.expand_dims(image, axis=2)
image = np.concatenate((image, image, image), axis=2)
image = image * 255.0
# image = image.astype(np.uint8)
image = image.astype("float32")
# if not self.detect_face:
# center = [450//2, 450//2+0]
# if self.center_shift != 0:
# center[0] += int(np.random.uniform(-self.center_shift,
# self.center_shift))
# center[1] += int(np.random.uniform(-self.center_shift,
# self.center_shift))
# scale = 1.8
# else:
# detected_faces = self.face_detector.detect_image(image)
# if len(detected_faces) > 0:
# box = detected_faces[0]
# left, top, right, bottom, _ = box
# center = [right - (right - left) / 2.0,
# bottom - (bottom - top) / 2.0]
# center[1] = center[1] - (bottom - top) * 0.12
# scale = (right - left + bottom - top) / 195.0
# else:
# center = [450//2, 450//2+0]
# scale = 1.8
# if self.center_shift != 0:
# shift = self.center * self.center_shift / 450
# center[0] += int(np.random.uniform(-shift, shift))
# center[1] += int(np.random.uniform(-shift, shift))
landmarks = self.landmarks_frame.iloc[idx, 4:140].values.astype('float64').reshape(-1, 2)
center = [self.landmarks_frame.iloc[idx,2],self.landmarks_frame.iloc[idx,3]]
scale = self.landmarks_frame.iloc[idx,1]
scale *= 1.25
###做随机的在线处理#############
if self.phase == "train":
print("I will deal with the data!")
scale = scale * (random.uniform(1 - self.scale_factor,
1 + self.scale_factor))
r = random.uniform(-self.rot_factor, self.rot_factor) \
if random.random() <= 0.6 else 0
if random.random() <= 0.5 and self.flip:
image = np.fliplr(image)
landmarks = fliplr_joints(landmarks, width=image.shape[1], dataset='300W')
center[0] = image.shape[1] - center[0]
new_image, new_landmarks = cv_crop(image, landmarks, center,
scale, 256, self.center_shift)
# if landmarks != []:
# new_image, new_landmarks = cv_crop(image, landmarks, center,
# scale, 256, self.center_shift)
# tries = 0
# while self.center_shift != 0 and tries < 5 and (np.max(new_landmarks) > 240 or np.min(new_landmarks) < 15):
# center = [450//2, 450//2+0]
# scale += 0.05
# center[0] += int(np.random.uniform(-self.center_shift,
# self.center_shift))
# center[1] += int(np.random.uniform(-self.center_shift,
# self.center_shift))
# new_image, new_landmarks = cv_crop(image, landmarks,
# center, scale, 256,
# self.center_shift)
# tries += 1
# if np.max(new_landmarks) > 250 or np.min(new_landmarks) < 5:
# center = [450//2, 450//2+0]
# scale = 2.25
# new_image, new_landmarks = cv_crop(image, landmarks,
# center, scale, 256,
# 100)
# print(np.min(new_landmarks), np.max(new_landmarks) )
# assert (np.min(new_landmarks) > 0 and np.max(new_landmarks) < 256), \
# "Landmarks out of boundary!"
# if not (np.min(new_landmarks) > 0 and np.max(new_landmarks) < 256):
# print(np.min(new_landmarks))
# print(np.max(new_landmarks))
# print(imgpath)
# print("Landmarks out of boundary!")
image = new_image
landmarks = new_landmarks
heatmap = np.zeros((self.num_lanmdkars, 64, 64))
for i in range(self.num_lanmdkars):
if landmarks[i][0] > 0:
heatmap[i] = draw_gaussian(heatmap[i], landmarks[i]/4.0+1, 1)
center = np.array(center)
sample = {"index": idx,'image': image, 'center': center, 'scale': scale, 'heatmap': heatmap, 'landmarks': landmarks}
if self.transform:
sample = self.transform(sample)
return sample