offset_px3 = 0 # (px3 - x1_) / side_len
offset_py3 = 0 # (py3 - y1_) / side_len
offset_px4 = 0 # (px4 - x1_) / side_len
offset_py4 = 0 # (py4 - y1_) / side_len
offset_px5 = 0 # (px5 - x1_) / side_len
offset_py5 = 0 # (py5 - y1_) / side_len
# 剪切下图片并进行大小缩放
# 抠图
face_crop = img.crop(crop_box)
# 按照人脸尺寸进行缩放：12/24/48
face_resize = face_crop((face_size, face_size),
Image.ANTIALIAS)
# 抠出来的框和原来的框计算IOU
iou = utils.iou(crop_box, np.array(boxes))[0]
# 调试后为0.6，调试前为0.65
if iou > 0.6:
positive_anno_file.write(
"positive/{0}.jpg {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14} {15}\n"
.format(positive_count, 1, offset_x1,
offset_y1, offset_x2, offset_y2,
offset_px1, offset_py1, offset_px2,
offset_py2, offset_px3, offset_py3,
offset_px4, offset_py4, offset_px5,
offset_py5))
positive_anno_file.flush() # flush：将缓存区的数据写入文件
face_resize.save(
os.path.join(
positive_image_dir,
"{0}.jpg".format(positive_count))) # 保存
def run(self, epoch):
# positive_label_txt = open(self.positive_label, "w")
negative_label_txt = open(self.negative_label, "w")
# part_label_txt = open(self.part_label, "w")
# positive_count = 0
negative_count = 0
# part_count = 0
for _ in range(epoch):
box_nano = open(self.src_anno_path, "r")
# landmark_anno = open(self.src_landmark_path, "r")
# for i, line in enumerate(open(self.src_anno_path, "r")):
for i, box_line in enumerate(box_nano):
if i < 2:
continue
strs = box_line.split()
# landmarks = landmark_line.split()
# print(strs, landmarks)
img_path = f"{self.src_image_path}/{strs[0]}"
img = Image.open(img_path)
# img.show()
x1 = int(strs[1])
y1 = int(strs[2])
w = int(strs[3])
h = int(strs[4])
x2 = x1 + w
y2 = y1 + h
if max(w, h) < 40 or x1 < 0 or y1 < 0 or w < 0 or h < 0:
continue
x1 = int(x1 + w * 0.12)
y1 = int(y1 + h * 0.1)
x2 = int(x1 + w * 0.9)
y2 = int(y1 + h * 0.85)
w = x2 - x1
h = y2 - y1
cx = int(x1 + (w / 2))
cy = int(y1 + (w / 2))
_cx = cx + np.random.randint(-w * 0.2, w * 0.2)
_cy = cy + np.random.randint(-h * 0.2, h * 0.2)
_w = w + np.random.randint(-w * 0.2, w * 0.2)
_h = h + np.random.randint(-h * 0.2, h * 0.2)
_x1 = int(_cx - (_w / 2))
_y1 = int(_cy - (_h / 2))
_x2 = int(_x1 + _w)
_y2 = int(_y1 + _h)
_x1_off = (_x1 - x1) / _w
_y1_off = (_y1 - y1) / _h
_x2_off = (_x2 - x2) / _w
_y2_off = (_y2 - y2) / _h
clip_img = img.crop([_x1, _y1, _x2, _y2])
clip_img = clip_img.resize((self.image_size, self.image_size))
iou = utils.iou(np.array([x1, y1, x2, y2]),
np.array([[_x1, _y1, _x2, _y2]]))
# if iou > 0.6 and positive_count <= 30000:
# clip_img.save(f"{self.positive_image_dir}/{positive_count}.jpg")
# positive_label_txt.write(
# "{0}.jpg {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14} {15}\n".format(
# positive_count, 1, _x1_off, _y1_off, _x2_off, _y2_off, _px1_off, _py1_off,
# _px2_off,_py2_off,_px3_off, _py3_off, _px4_off, _py4_off, _px5_off, _py5_off
# ))
# positive_label_txt.flush()
# positive_count += 1
# elif iou > 0.4 and part_count <= 30000:
# clip_img.save(f"{self.part_image_dir}/{part_count}.jpg")
# part_label_txt.write(
# "{0}.jpg {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14} {15}\n".format(
# part_count, 2, _x1_off, _y1_off, _x2_off, _y2_off, _px1_off, _py1_off,
# _px2_off, _py2_off, _px3_off, _py3_off, _px4_off, _py4_off, _px5_off, _py5_off
# ))
# part_label_txt.flush()
# part_count += 1
if iou < 0.3 and negative_count <= 90000:
clip_img.save(
f"{self.negative_image_dir}/{negative_count}.jpg")
negative_label_txt.write(
"{0}.jpg {1} {2} {3} {4} {5} \n".format(
negative_count, 0, 0, 0, 0, 0))
negative_label_txt.flush()
negative_count += 1
w, h = img.size
_x1, _y1 = np.random.randint(0, w), np.random.randint(0, h)
_w, _h = np.random.randint(0, w - _x1), np.random.randint(
0, h - _y1)
_x2, _y2 = _x1 + _w, _y1 + _h
clip_img = img.crop([_x1, _y1, _x2, _y2])
clip_img = clip_img.resize((self.image_size, self.image_size))
iou = utils.iou(np.array([x1, y1, x2, y2]),
np.array([[_x1, _y1, _x2, _y2]]))
# if iou > 0.6 and positive_count <= 30000:
# clip_img.save(f"{self.positive_image_dir}/{positive_count}.jpg")
# positive_label_txt.write(
# "{0}.jpg {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14} {15}\n".format(
# positive_count, 1, _x1_off, _y1_off, _x2_off, _y2_off, _px1_off, _py1_off,
# _px2_off, _py2_off, _px3_off, _py3_off, _px4_off, _py4_off, _px5_off, _py5_off
# ))
# positive_label_txt.flush()
# positive_count += 1
# elif iou > 0.4 and part_count <= 30000:
# clip_img.save(f"{self.part_image_dir}/{part_count}.jpg")
# part_label_txt.write(
# "{0}.jpg {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14} {15}\n".format(
# part_count, 2, _x1_off, _y1_off, _x2_off, _y2_off, _px1_off, _py1_off,
# _px2_off, _py2_off, _px3_off, _py3_off, _px4_off, _py4_off, _px5_off, _py5_off
# ))
# part_label_txt.flush()
# part_count += 1
if iou < 0.3 and negative_count <= 90000:
clip_img.save(
f"{self.negative_image_dir}/{negative_count}.jpg")
negative_label_txt.write(
"{0}.jpg {1} {2} {3} {4} {5}\n".format(
negative_count, 0, 0, 0, 0, 0))
negative_label_txt.flush()
negative_count += 1
# positive_label_txt.close()
negative_label_txt.close()
def run(self):
positive_meta = open(
os.path.join(self.net_data_folder, 'positive_meta.txt'), 'w')
negative_meta = open(
os.path.join(self.net_data_folder, 'negative_meta.txt'), "w")
part_meta = open(os.path.join(self.net_data_folder, 'part_meta.txt'),
'w')
positive_count = 0
negative_count = 0
part_count = 0
for i, item in enumerate(self.metadata):
img_path = item['file_name']
img = Image.open(img_path)
boxes = np.array(item['meta_data'])[:, :4]
boxes = boxes[boxes[:, 2] >= 0]
boxes = boxes[boxes[:, 3] >= 0]
boxes[:, 2] += boxes[:, 0]
boxes[:, 3] += boxes[:, 1]
width, height = img.size
# print(img.size,boxes.shape)
for box in boxes:
x1, y1, x2, y2 = box
w = x2 - x1 + 1
h = y2 - y1 + 1
if max(w, h) < 40 or x1 < 0 or y1 < 0:
continue
size = np.random.randint(int(min(w, h) * 0.8),
np.ceil(1.25 * max(w, h)))
delta_x = np.random.randint(-w * 0.2, w * 0.2)
delta_y = np.random.randint(-h * 0.2, h * 0.2)
nx1 = int(max(x1 + w / 2 + delta_x - size / 2, 0))
ny1 = int(max(y1 + h / 2 + delta_y - size / 2, 0))
nx2 = nx1 + size
ny2 = ny1 + size
if nx2 > width or ny2 > height:
continue
crop_box = np.array([nx1, ny1, nx2, ny2])
offset_x1 = (x1 - nx1) / float(size)
offset_y1 = (y1 - ny1) / float(size)
offset_x2 = (x2 - nx2) / float(size)
offset_y2 = (y2 - ny2) / float(size)
crop_img = img.crop(crop_box)
resize_img = crop_img.resize((self.crop_size, self.crop_size))
_box = np.array([x1, y1, x2, y2])
iou = utils.iou(_box, np.array([[nx1, ny1, nx2, ny2]]))
if iou >= 0.65 and positive_count < 30000:
positive_count += 1
positive_meta.write(
f"{positive_count}.jpg {1} {offset_x1} {offset_y1} {offset_x2} {offset_y2} \n"
)
resize_img.save(
f"{self.positive_dest}/{positive_count}.jpg")
positive_meta.flush()
if iou > 0.4 and part_count < 30000:
part_count += 1
part_meta.write(
f"{part_count}.jpg {2} {offset_x1} {offset_y1} {offset_x2} {offset_y2} \n"
)
resize_img.save(f"{self.part_dest}/{part_count}.jpg")
part_meta.flush()
size = np.random.randint(self.crop_size,
min(width, height) + 1 / 2)
delta_x = np.random.randint(max(-size, -x1), w)
delta_y = np.random.randint(max(-size, -y1), h)
nx1 = max(x1 + delta_x, 0)
ny1 = max(y1 + delta_y, 0)
nx2 = nx1 + size
ny2 = ny1 + size
if nx2 > width or ny2 > height:
continue
crop_box = np.array([nx1, ny1, nx2, ny2])
crop_img = img.crop(crop_box)
resize_img = crop_img.resize((self.crop_size, self.crop_size))
_box = np.array([x1, y1, x2, y2])
iou = utils.iou(_box, np.array([[nx1, ny1, nx2, ny2]]))
if iou < 0.3 and negative_count < 90000:
negative_count += 1
negative_meta.write(
f"{negative_count}.jpg {0} {0} {0} {0} {0} \n")
resize_img.save(
f"{self.negative_dest}/{negative_count}.jpg")
negative_meta.flush()
positive_meta.close()
negative_meta.close()
part_meta.close()
x2_ = cx_ + side_len / 2
y2_ = cy_ + side_len / 2
# 计算每个框的偏移值
offset_x1 = (x1 - x1_) / side_len
offset_y1 = (y1 - y1_) / side_len
offset_x2 = (x2 - x2_) / side_len
offset_y2 = (y2 - y2_) / side_len
crop_box = np.array([x1_, y1_, x2_, y2_])
# 剪切图片
img_crop = img.crop(crop_box)
img_resize = img_crop.resize((face_size, face_size))
iou = utils.iou(crop_box, boxes)
# if iou > 0.65: # 正样本
# positive_anno_dir.write(
# 'positive/{0}.jpg {1} {2} {3} {4} {5} {6}\n'.format(positive_count, 1, iou[0], offset_x1,
# offset_y1,
# offset_x2, offset_y2))
# positive_anno_dir.flush()
# img_resize.save(os.path.join(positive_img_dir, '{}.jpg'.format(positive_count)))
# positive_count += 1
if iou < 0.5 and iou > 0.3: # 部分样本
part_anno_dir.write(
'part/{0}.jpg {1} {2} {3} {4} {5} {6}\n'.format(part_count, 2, iou[0], offset_x1, offset_y1,
offset_x2, offset_y2))
part_anno_dir.flush()
def generate_labels(self, args):
"""
Overwriting parent class to generate action proposal labels
"""
self.training_ids = list(self.train.keys())
self.validation_ids = list(self.val.keys())
print("| Generating labels for action proposals")
label_dataset = h5py.File(args.labels, 'w')
prop_captured = []
prop_pos_examples = []
data = []
data.extend(self.training_ids)
data.extend(self.validation_ids)
train_sz = len(self.training_ids)
bar = progressbar.ProgressBar(maxval=len(data)).start()
for progress, video_id in enumerate(data):
features = self.features[video_id]['c3d_features']
nfeats = features.shape[0]
if progress < train_sz:
duration = self.train[video_id]['duration']
timestamps = self.train[video_id]['timestamps']
else:
duration = self.val[video_id]['duration']
timestamps = self.val[video_id]['timestamps']
featstamps = [
self.timestamp_to_featstamp(x, nfeats, duration)
for x in timestamps
]
nb_prop = len(featstamps)
for i in range(nb_prop):
if (featstamps[nb_prop - i - 1][1] -
featstamps[nb_prop - i -
1][0]) > args.K / args.iou_threshold:
# we discard these proposals since they will not be captured for this value of K
del featstamps[nb_prop - i - 1]
if len(featstamps) == 0:
if len(timestamps) == 0:
# no proposals in this video
prop_captured += [-1.]
else:
# no proposals captured in this video since all have a length above threshold
prop_captured += [0.]
if progress < train_sz:
self.training_ids.remove(video_id)
else:
self.validation_ids.remove(video_id)
continue
labels = np.zeros((nfeats, args.K))
gt_captured = []
for t in range(nfeats):
for k in range(args.K):
iou, gt_index = utils.iou([t - k, t + 1],
featstamps,
return_index=True)
if iou >= args.iou_threshold:
labels[t, k] = 1
gt_captured += [gt_index]
prop_captured += [
1. * len(np.unique(gt_captured)) / len(timestamps)
]
if progress < train_sz:
prop_pos_examples += [np.sum(labels, axis=0) * 1. / nfeats]
video_dataset = label_dataset.create_dataset(video_id,
(nfeats, args.K),
dtype='f')
video_dataset[...] = labels
bar.update(progress)
split_ids = {
'training': self.training_ids,
'validation': self.validation_ids,
'testing': self.testing_ids,
'pos_prop_weight': np.array(prop_pos_examples).mean(
axis=0).tolist() # this will be used to compute the loss
}
json.dump(split_ids, open(args.vid_ids, 'w'))
self.compute_proposals_stats(np.array(prop_captured))
bar.finish()