def check_box_and_cropface(orig_img_path, channel_first=False):
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_box(
orig_img_path, channel_first=channel_first, mc_manager=None)
AU_couple = get_zip_ROI_AU()
already_couple = set()
# cropped_face = np.transpose(cropped_face, (2, 0, 1))
# cropped_face, params = transforms.random_flip(
# cropped_face, x_random=True, return_param=True)
# cropped_face = np.transpose(cropped_face, (1, 2, 0))
i = 0
for AU, box_ls in AU_mask_dict.items():
current_AU_couple = AU_couple[AU]
if current_AU_couple in already_couple:
continue
already_couple.add(current_AU_couple)
for box in box_ls:
box = np.asarray([box])
box = transforms.flip_bbox(box, (512, 512), x_flip=False)
x_min, y_min = box[0][1], box[0][0]
x_max, y_max = box[0][3], box[0][2]
print(box)
cp_croped = cropped_face.copy()
cv2.rectangle(cp_croped, (x_min, y_min), (x_max, y_max),
(0, 255, 0), 1)
cv2.imwrite(
"/home2/mac/test1/AU_{0}_{1}.png".format(
",".join(current_AU_couple), i), cp_croped)
i += 1
print(i)
def delegate_mask_crop(img_path, channal_first, queue):
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channal_first)
print("crop {} done".format(img_path))
queue.put((img_path, cropped_face, AU_mask_dict), block=True)
except IndexError:
pass
def get_example(self, i):
rgb_path, flow_path, AU_set, database_name = self.result_data[i]
video_seq_id = self.extract_sequence_key(rgb_path)
# note that batch now is mix of T and batch_size, we must be reshape later
try:
rgb_face, flow_face, bbox, label = self.AU_image_dataset.get_from_entry(
flow_path, rgb_path, AU_set, database_name)
assert bbox.shape[0] == label.shape[0]
if bbox.shape[0] != config.BOX_NUM[database_name]:
print("found one error image: {0} box_number:{1}".format(
rgb_path, bbox.shape[0]))
bbox = bbox.tolist()
label = label.tolist()
if len(bbox) > config.BOX_NUM[database_name]:
all_del_idx = []
for idx, box in enumerate(bbox):
if FaceMaskCropper.calculate_area(*box) / float(
config.IMG_SIZE[0] *
config.IMG_SIZE[1]) < 0.01:
all_del_idx.append(idx)
for del_idx in all_del_idx:
del bbox[del_idx]
del label[del_idx]
while len(bbox) < config.BOX_NUM[database_name]:
index = 0
bbox.insert(0, bbox[index])
label.insert(0, label[index])
while len(bbox) > config.BOX_NUM[database_name]:
del bbox[-1]
del label[-1]
bbox = np.stack(bbox)
label = np.stack(label)
except IndexError:
print("image path : {} not get box".format(rgb_path))
label = np.zeros(len(config.AU_SQUEEZE), dtype=np.int32)
for AU in AU_set:
np.put(label, config.AU_SQUEEZE.inv[AU], 1)
if self.paper_report_label_idx:
label = label[self.paper_report_label_idx]
rgb_whole_image = np.transpose(
cv2.resize(cv2.imread(rgb_path), config.IMG_SIZE), (2, 0, 1))
flow_whole_image = np.transpose(
cv2.resize(cv2.imread(flow_path), config.IMG_SIZE), (2, 0, 1))
whole_bbox = np.tile(
np.array(
[1, 1, config.IMG_SIZE[1] - 2, config.IMG_SIZE[0] - 2],
dtype=np.float32), (config.BOX_NUM[database_name], 1))
whole_label = np.tile(label, (config.BOX_NUM[database_name], 1))
return rgb_whole_image, flow_whole_image, whole_bbox, whole_label, rgb_path
assert bbox.shape[0] == config.BOX_NUM[database_name], bbox.shape[0]
if self.paper_report_label_idx:
label = label[:, self.paper_report_label_idx]
return rgb_face, flow_face, bbox, label, rgb_path
def generate_AUCouple_ROI_mask_image(database_name, img_path):
adaptive_AU_database(database_name)
global MASK_COLOR
mask_color_lst = []
for color in MASK_COLOR:
mask_color_lst.append(color_bgr(color))
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True,
color_bgr(RED),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr, axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
idx = 0
gen_face_lst = dict()
AU_couple_mask = dict()
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in already_fill_AU:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
AU_couple_mask[AU_couple] = mask
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = random.choice(mask_color_lst)
idx += 1
new_face = cv2.addWeighted(cropped_face, 0.75, color_mask, 0.25, 0)
gen_face_lst[AU_couple] = new_face
return gen_face_lst, AU_couple_mask
def get_example(self, i):
'''
Returns a color image and bounding boxes. The image is in CHW format.
The returned image is RGB.
:param i: the index of the example
:return: tuple of an image and its all bounding box
'''
if i > len(self.result_data):
raise IndexError("Index too large")
img_path, from_img_path, AU_set, database_name = self.result_data[i]
if not os.path.exists(img_path):
raise IndexError("image file_path: {} not exist!".format(img_path))
read_img_path = img_path if from_img_path == "#" else from_img_path
rgb_img_path = config.RGB_PATH[
database_name] + os.path.sep + os.path.sep.join(
read_img_path.split("/")[-3:])
flow_img_path = config.FLOW_PATH[
database_name] + os.path.sep + os.path.sep.join(
read_img_path.split("/")[-3:])
key_prefix = self.database + "@{0}".format(self.img_resolution) + "|"
if self.pretrained_target is not None and len(
self.pretrained_target) > 0:
key_prefix = self.pretrained_target + "|"
if self.read_type == "rgb":
read_img_path = rgb_img_path
elif self.read_type == "flow":
read_img_path = flow_img_path
try:
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
read_img_path,
read_img_path,
channel_first=True,
mc_manager=self.mc_manager,
key_prefix=key_prefix)
except IndexError:
print("error in crop face: {}".format(read_img_path))
cropped_face = cv2.imread(read_img_path, cv2.IMREAD_COLOR)
cropped_face = cv2.resize(cropped_face, config.IMG_SIZE)
cropped_face = np.transpose(cropped_face, axes=(2, 0, 1))
label = np.zeros(shape=len(config.AU_SQUEEZE), dtype=np.int32)
for AU in AU_set:
if AU in config.AU_SQUEEZE.inv:
label[config.AU_SQUEEZE.inv[AU]] = 1
return cropped_face, label
def generate_AUCouple_ROI_mask_image(self, database_name, img_path,
roi_activate):
adaptive_AU_database(database_name)
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True, (0, 0, 255),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr,
axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
AUCouple_face_dict = dict()
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in already_fill_AU or AU_couple not in roi_activate:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = (199, 21, 133)
new_face = cv2.add(cropped_face, color_mask)
AUCouple_face_dict[AU_couple] = new_face
return AUCouple_face_dict
def collect_box_coordinate(image_path, mc_manager, database):
au_couple_dict = get_zip_ROI_AU()
AU_group_box_coodinate = defaultdict(list) # AU_group => box_list
key_prefix = database + "@512|"
try:
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
image_path,
image_path,
channel_first=True,
mc_manager=mc_manager,
key_prefix=key_prefix)
except IndexError:
return AU_group_box_coodinate
for AU, box_list in AU_box_dict.items():
AU_couple = au_couple_dict[AU]
if AU_couple in AU_group_box_coodinate:
continue
new_box_list = [list(box) for box in box_list]
new_box_list.sort(key=lambda e: e[1])
AU_group_box_coodinate[AU_couple].extend(new_box_list)
return AU_group_box_coodinate
def stats_AU_group_area(image_path, mc_manager, database):
au_couple_dict = get_zip_ROI_AU()
AU_group_box_area = dict()
key_prefix = database + "|"
try:
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
image_path,
image_path,
channel_first=True,
mc_manager=mc_manager,
key_prefix=key_prefix)
except IndexError:
return AU_group_box_area
for AU, box_list in AU_box_dict.items():
AU_couple = au_couple_dict[AU]
tot_area = 0.
for box in box_list:
y_min, x_min, y_max, x_max = box
area = (x_max - x_min) * (y_max - y_min)
tot_area += area
tot_area /= len(box_list)
AU_group_box_area[AU_couple] = tot_area
return AU_group_box_area
def generate_mask_contain_img(database_name, img_path):
adaptive_AU_database(database_name)
mask_color = {}
for parent_color, child_color in MASK_CONTAIN.items():
mask_color[color_bgr(parent_color)] = color_bgr(child_color)
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
AU_couple_child = get_AU_couple_child(AU_couple_dict)
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True,
color_bgr(RED),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr, axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
gen_face_lst = dict()
all_child_set = set()
for child_set in AU_couple_child.values():
for child in child_set:
all_child_set.add(child)
new_face = np.zeros_like(cropped_face)
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in all_child_set:
continue
if AU_couple in already_fill_AU:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
child_AU_set = AU_couple_child[AU_couple]
color_parent = list(MASK_CONTAIN.keys())[0]
color_child = MASK_CONTAIN[color_parent]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = color_parent
# cv2.addWeighted(color_mask,0.5, color_mask,1-0.5,0,color_mask)
if np.any(new_face):
cropped_face = new_face
cv2.addWeighted(cropped_face, 1, color_mask, 0.3, 0, new_face, -1)
for child_AU in child_AU_set:
if child_AU in already_fill_AU:
continue
already_fill_AU.add(child_AU)
mask = AU_mask_dict[child_AU[0]]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = random.choice(color_child)
cv2.addWeighted(new_face, 1, color_mask, 0.5, 0, new_face, -1)
return new_face
def get_example(self, i):
'''
Returns a color image and bounding boxes. The image is in CHW format.
The returned image is RGB.
:param i: the index of the example
:return: tuple of an image and its all bounding box
'''
if i > len(self.result_data):
raise IndexError("Index too large")
img_path, from_img_path, AU_intensity, database_name = self.result_data[
i]
if not os.path.exists(img_path):
raise IndexError("image file_path: {} not exist!".format(img_path))
try:
# print("begin fetch cropped image and bbox {}".format(img_path))
read_img_path = img_path if from_img_path == "#" else from_img_path
rgb_img_path = config.RGB_PATH[
self.database] + os.path.sep + os.path.sep.join(
read_img_path.split("/")[-3:])
key_prefix = self.database + "@{}".format(
self.img_resolution) + "|"
if self.pretrained_target is not None and len(
self.pretrained_target) > 0:
key_prefix = self.pretrained_target + "|"
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
read_img_path,
rgb_img_path,
channel_first=True,
mc_manager=self.mc_manager,
key_prefix=key_prefix)
except IndexError:
print("crop image error:{}".format(img_path))
face = np.transpose(
cv2.resize(cv2.imread(img_path), config.IMG_SIZE), (2, 0, 1))
whole_bbox = np.tile(
np.array(
[1, 1, config.IMG_SIZE[1] - 1, config.IMG_SIZE[0] - 1],
dtype=np.float32), (config.BOX_NUM[database_name], 1))
whole_label = np.tile(AU_intensity,
(config.BOX_NUM[database_name], 1))
return face, whole_bbox, whole_label
current_AU_couple = defaultdict(
dict) # key = AU couple, value = {出现的AU: intensity}
couple_box_dict = OrderedDict() # key= AU couple
for idx, intensity in enumerate(AU_intensity):
AU = str(config.AU_INTENSITY_DICT[idx])
if intensity > 0:
try:
current_AU_couple[self.au_couple_dict[AU]][AU] = intensity
except KeyError:
print(list(self.au_couple_dict.keys()), AU)
raise
for AU, box_list in sorted(AU_box_dict.items(),
key=lambda e: int(e[0])):
AU = str(AU)
couple_box_dict[
self.au_couple_dict[AU]] = box_list # 所以这一步会把脸上有的，没有的AU都加上
label = [
] # one box may have multiple labels. so each entry is 10101110 binary code
bbox = [] # AU = 0背景的box是随机取的
self.assign_label(couple_box_dict, current_AU_couple, bbox, label)
bbox = np.stack(bbox).astype(np.float32)
label = np.stack(label).astype(np.int32)
assert bbox.shape[0] == label.shape[0]
return cropped_face, bbox, label
def main():
parser = argparse.ArgumentParser(
description='Space Time Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
nargs='+',
type=int,
help='GPU ID, multiple GPU split by space')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='end_to_end_result',
help='Output directory')
parser.add_argument('--trainval', default='train', help='train/test')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=1)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--need_validate',
action='store_true',
help='do or not validate during training')
parser.add_argument('--mean',
default=config.ROOT_PATH +
"BP4D/idx/mean_no_enhance.npy",
help='image mean .npy file')
parser.add_argument('--backbone',
default="mobilenet_v1",
help="vgg/resnet101/mobilenet_v1 for train")
parser.add_argument('--optimizer',
default='RMSprop',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model',
default='mobilenet_v1',
help='imagenet/mobilenet_v1/resnet101/*.npz')
parser.add_argument('--pretrained_model_args',
nargs='+',
type=float,
help='you can pass in "1.0 224" or "0.75 224"')
parser.add_argument('--spatial_edge_mode',
type=SpatialEdgeMode,
choices=list(SpatialEdgeMode),
help='1:all_edge, 2:configure_edge, 3:no_edge')
parser.add_argument(
'--temporal_edge_mode',
type=TemporalEdgeMode,
choices=list(TemporalEdgeMode),
help='1:rnn, 2:attention_block, 3.point-wise feed forward(no temporal)'
)
parser.add_argument("--bi_lstm",
action="store_true",
help="whether to use bi-lstm as Edge/Node RNN")
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--layers", type=int, default=1)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--use_paper_num_label",
action="store_true",
help="only to use paper reported number of labels"
" to train")
parser.add_argument("--previous_frame", type=int, default=50)
parser.add_argument("--sample_frame", '-sample', type=int, default=25)
parser.add_argument(
"--snap_individual",
action="store_true",
help="whether to snapshot each individual epoch/iteration")
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
args = parser.parse_args()
os.makedirs(args.pid, exist_ok=True)
os.makedirs(args.out, exist_ok=True)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
print('GPU: {}'.format(",".join(list(map(str, args.gpu)))))
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
train_data = AUDataset(
database=args.database,
fold=args.fold,
split_name=args.trainval,
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False,
)
result_data = [
img_path
for img_path, AU_set, current_database_name in train_data.result_data
if args.database + "|" + img_path not in mc_manager
]
sub_list = split_list(result_data, len(result_data) // 100)
for img_path_lst in sub_list:
with Pool(processes=50) as pool:
input_list = [(img_path, None, None) for img_path in img_path_lst]
result =\
pool.starmap(parallel_landmark_and_conn_component, input_list)
pool.close()
pool.join()
for img_path, AU_box_dict, landmark_dict, box_is_whole_image in result:
key_prefix = args.database + "|"
key = key_prefix + img_path
orig_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
new_face, rect = FaceMaskCropper.dlib_face_crop(
orig_img, landmark_dict)
print("write {}".format(key))
if mc_manager is not None and key not in mc_manager:
save_dict = {
"landmark_dict": landmark_dict,
"AU_box_dict": AU_box_dict,
"crop_rect": rect
}
mc_manager.set(key, save_dict)
def main():
parser = argparse.ArgumentParser(
description='generate Graph desc file script')
parser.add_argument('--mean',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument("--image",
default='C:/Users/machen/Downloads/tmp/face.jpg')
parser.add_argument(
"--model", default="C:/Users/machen/Downloads/tmp/BP4D_3_fold_1.npz")
parser.add_argument("--pretrained_model_name",
'-premodel',
default='resnet101')
parser.add_argument('--database', default='BP4D', help='Output directory')
parser.add_argument('--device',
default=0,
type=int,
help='GPU device number')
args = parser.parse_args()
adaptive_AU_database(args.database)
if args.pretrained_model_name == "resnet101":
faster_rcnn = FasterRCNNResnet101(
n_fg_class=len(config.AU_SQUEEZE),
pretrained_model="resnet101",
mean_file=args.mean,
use_lstm=False,
extract_len=1000
) # 可改为/home/machen/face_expr/result/snapshot_model.npz
elif args.pretrained_model_name == "vgg":
faster_rcnn = FasterRCNNVGG16(n_fg_class=len(config.AU_SQUEEZE),
pretrained_model="imagenet",
mean_file=args.mean,
use_lstm=False,
extract_len=1000)
if os.path.exists(args.model):
print("loading pretrained snapshot:{}".format(args.model))
chainer.serializers.load_npz(args.model, faster_rcnn)
if args.device >= 0:
faster_rcnn.to_gpu(args.device)
chainer.cuda.get_device_from_id(int(args.device)).use()
heatmap_gen = HeatMapGenerator(np.load(args.model), use_relu=True)
if args.device >= 0:
heatmap_gen.to_gpu(args.device)
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
args.image, args.image, channel_first=True)
au_couple_dict = get_zip_ROI_AU()
au_couple_child = get_AU_couple_child(
au_couple_dict) # AU couple tuple => child fetch list
au_couple_box = dict() # value is box (4 tuple coordinate) list
for AU, AU_couple in au_couple_dict.items():
au_couple_box[AU_couple] = AU_box_dict[AU]
box_lst = []
roi_no_AU_couple_dict = dict()
roi_no = 0
for AU_couple, couple_box_lst in au_couple_box.items():
box_lst.extend(couple_box_lst)
for _ in couple_box_lst:
roi_no_AU_couple_dict[roi_no] = AU_couple
roi_no += 1
box_lst = np.asarray(box_lst)
cropped_face = cropped_face.astype(np.float32)
orig_face = cropped_face
cropped_face = faster_rcnn.prepare(
cropped_face) # substract mean pixel value
box_lst = box_lst.astype(np.float32)
orig_box_lst = box_lst
batch = [
(cropped_face, box_lst),
]
cropped_face, box_lst = concat_examples(
batch, args.device) # N,3, H, W, ; N, F, 4
if box_lst.shape[1] != config.BOX_NUM[args.database]:
print("error box num {0} != {1}".format(box_lst.shape[1],
config.BOX_NUM[args.database]))
return
with chainer.no_backprop_mode(), chainer.using_config("train", False):
cropped_face = chainer.Variable(cropped_face)
box_lst = chainer.Variable(box_lst)
roi_preds, _ = faster_rcnn.predict(cropped_face, box_lst) # R, 22
roi_feature_maps = faster_rcnn.extract(orig_face, orig_box_lst,
'res5') # R, 2048 7,7
roi_images = []
box_lst = box_lst[0].data.astype(np.int32)
for box in box_lst:
y_min, x_min, y_max, x_max = box
roi_image = orig_face[:, y_min:y_max + 1,
x_min:x_max + 1] # N, 3, roi_H, roi_W
roi_images.append(roi_image) # list of N, 3, roi_H, roi_W
cmap = plt.get_cmap('jet')
# image_activate_map = np.zeros((cropped_face.shape[2], cropped_face.shape[3]), dtype=np.float32)
for box_id, (roi_image, roi_feature_map) in enumerate(
zip(roi_images, roi_feature_maps)):
y_min, x_min, y_max, x_max = box_lst[box_id]
# 22, roi_h, roi_w, 3
xp = chainer.cuda.get_array_module(roi_feature_map)
roi_feature_map = xp.expand_dims(roi_feature_map, 0)
# class_roi_overlay_img = 22, roi_h, roi_w
class_roi_activate_img = heatmap_gen.generate_activate_roi_map(
roi_feature_map, (y_max - y_min + 1, x_max - x_min + 1))
roi_pred = roi_preds[box_id] # 22
# choice_activate_map = np.zeros((y_max-y_min+1, x_max-x_min+1), dtype=np.float32)
# use_choice = False
if len(np.nonzero(roi_pred)
[0]) > 0: # TODO : 还要做做 class的选择，以及 heatmap采用cv2.add的模式相加
class_idx = random.choice(np.nonzero(roi_pred)[0])
AU = config.AU_SQUEEZE[class_idx]
print(AU)
choice_activate_map = class_roi_activate_img[
class_idx] # roi_h, roi_w
activation_color_map = np.round(
cmap(choice_activate_map)[:, :, :3] * 255).astype(np.uint8)
overlay_img = roi_images[
box_id] / 2 + activation_color_map.transpose(2, 0, 1) / 2
overlay_img = np.transpose(overlay_img,
(1, 2, 0)).astype(np.uint8)
vis_img = cv2.cvtColor(overlay_img, cv2.COLOR_RGB2BGR)
cv2.imshow("new", vis_img)
cv2.waitKey(0)
def read_DISFA_video_label(output_dir,
is_binary_AU,
is_need_adaptive_AU_relation=False,
force_generate=True,
proc_num=10,
cut=False,
train_subject=None):
mgr = mp.Manager()
queue = mgr.Queue(maxsize=20000)
for orientation in ["Left", "Right"]:
if is_need_adaptive_AU_relation:
adaptive_AU_relation(
) # delete AU relation pair occur in same facial region
au_couple_dict = get_zip_ROI_AU()
au_couple_child_dict = get_AU_couple_child(au_couple_dict)
DISFA_base_dir = config.DATA_PATH["DISFA"]
label_file_dir = DISFA_base_dir + "/ActionUnit_Labels/"
img_folder = DISFA_base_dir + "/Img_{}Camera".format(orientation)
for video_name in os.listdir(label_file_dir): # each file is a video
is_train = True if video_name in train_subject else False
if not force_generate:
prefix = "train" if is_train else "test"
target_file_path = output_dir + os.sep + prefix + os.sep + video_name + "_" + orientation + ".npz"
if os.path.exists(target_file_path):
continue
resultdict = {}
if proc_num > 1:
pool = mp.Pool(processes=proc_num)
procs = 0
one_file_name = os.listdir(label_file_dir + os.sep +
video_name)[0]
with open(
label_file_dir + os.sep + video_name + os.sep +
one_file_name, "r") as file_obj:
for idx, line in enumerate(file_obj):
line = line.strip()
if line:
frame = line.split(",")[0]
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
pool.apply_async(func=delegate_mask_crop,
args=(img_path, True, queue))
procs += 1
for i in range(procs):
try:
entry = queue.get(block=True, timeout=60)
resultdict[entry[0]] = (entry[1], entry[2])
except Exception:
print("queue block time out")
break
pool.close()
pool.join()
del pool
else: # only one process
one_file_name = os.listdir(label_file_dir + os.sep +
video_name)[0]
with open(
label_file_dir + os.sep + video_name + os.sep +
one_file_name, "r") as file_obj:
for idx, line in enumerate(file_obj):
line = line.strip()
if line:
frame = line.split(",")[0]
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, True)
resultdict[img_path] = (cropped_face,
AU_mask_dict)
except IndexError:
pass
frame_label = dict()
video_info = []
video_img_path_set = set()
for file_name in os.listdir(
label_file_dir + os.sep +
video_name): # each file is one AU ( video file )
AU = file_name[file_name.index("au") + 2:file_name.rindex(".")]
with open(
label_file_dir + os.sep + video_name + os.sep +
file_name, "r") as file_obj:
for line in file_obj:
frame = int(line.split(",")[0])
AU_intensity = int(line.split(",")[1])
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
video_img_path_set.add((frame, img_path))
if frame not in frame_label:
frame_label[frame] = set()
if AU_intensity >= 1: # NOTE that we apply AU_intensity >= 1
frame_label[int(frame)].add(AU) #存储str类型
for frame, img_path in sorted(video_img_path_set,
key=lambda e: int(e[0])):
if img_path not in resultdict:
continue
AU_set = frame_label[frame] # it is whole image's AU set
if cut and len(AU_set) == 0:
continue
cropped_face, AU_mask_dict = resultdict[img_path]
all_couple_mask_dict = OrderedDict()
for AU in sorted(map(
int, config.AU_ROI.keys())): # ensure same order
all_couple_mask_dict[au_couple_dict[str(
AU)]] = AU_mask_dict[str(AU)]
all_labels = list() # 开始拼接all_labels
for AU_couple in all_couple_mask_dict.keys(
): # 顺序与all_couple_mask_dict一致
child_AU_couple_list = au_couple_child_dict[AU_couple]
AU_couple = set(AU_couple)
for child_AU_couple in child_AU_couple_list:
AU_couple.update(
child_AU_couple) # combine child region's AU
if not is_binary_AU: # in CRF, CRF模式需要将同一个区域的多个AU用逗号分隔，拼接
concat_AU = []
for AU in AU_couple:
if AU in AU_set: # AU_set 存储真实AU(ground truth label):str类型
concat_AU.append(AU)
if len(concat_AU) == 0:
all_labels.append(
"0") # 若该区域压根没有任何AU出现，为了让只支持单label的CRF工作，用0来代替
else:
all_labels.append(",".join(sorted(concat_AU)))
else: # convert to np.array which is AU_bin
AU_bin = np.zeros(len(config.AU_SQUEEZE)).astype(
np.uint8)
for AU in AU_couple:
if AU in AU_set: # judge if this region contain which subset of whole image's AU_set
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], 1)
all_labels.append(tuple(AU_bin))
video_info.append({
"frame": frame,
"cropped_face": cropped_face,
"all_couple_mask_dict": all_couple_mask_dict,
"all_labels": all_labels,
"video_id": video_name + "_" + orientation
})
resultdict.clear()
if video_info:
yield video_info, video_name
else:
print(
"error in file:{} no video found".format(video_name + "_" +
orientation))
def read_BP4D_video_label(output_dir,
is_binary_AU,
is_need_adaptive_AU_relation=False,
force_generate=True,
proc_num=10,
cut=False,
train_subject=None):
'''
:param
output_dir : 用于检查如果目标的文件已经存在，那么就不再生成
is_binary_AU:
True --> return AU_binary 01010100
False --> used for CRF mode: single true AU label CRF/ or AU combination separate by comma
:yield: 每个视频video收集齐了yield回去，视频中每一帧返回3部分：
1. "img_path": /path/to/image
1."all_couple_mask_dict": 是OrderedDict，包含所有区域的mask，不管AU是不是+1，还是-1(不管AU出现没出现)，key是AU_couple，来自于au_couple_dict = get_zip_ROI_AU()
2."labels": 是list，index与all_couple_mask_dict一致，其中每个label
要么是binary形式01010110，
要么是3,4（由于一块位置可以发生多个AU，因此可以用逗号隔开的字符串来返回），根据is_binary_AU返回不同的值
'''
mgr = mp.Manager()
queue = mgr.Queue(maxsize=20000)
if is_need_adaptive_AU_relation:
adaptive_AU_relation(
) # delete AU relation pair occur in same facial region
au_couple_dict = get_zip_ROI_AU()
au_couple_child_dict = get_AU_couple_child(
au_couple_dict) # AU_couple => list of child AU_couple
# if need_translate_combine_AU ==> "mask_path_dict":{(2,3,4): /pathtomask} convert to "mask_path_dict":{110: /pathtomask}
# each is dict : {"img": /path/to/img, "mask_path_dict":{(2,3,4): /pathtomask}, }
BP4D_base_dir_path = config.DATA_PATH["BP4D"]
label_file_dir = BP4D_base_dir_path + "/AUCoding/"
for file_name in os.listdir(label_file_dir): # each file is a video
subject_name = file_name[:file_name.index("_")]
sequence_name = file_name[file_name.index("_") +
1:file_name.rindex(".")]
is_train = True if subject_name in train_subject else False
if not force_generate:
prefix = "train" if is_train else "test"
target_file_path = output_dir + os.sep + prefix + os.sep + subject_name + "_" + sequence_name + ".npz"
if os.path.exists(target_file_path):
continue
resultdict = {}
if proc_num > 1:
one_image_path = os.listdir(config.RGB_PATH["BP4D"] + os.sep +
subject_name + os.sep +
sequence_name)[0]
zfill_len = len(one_image_path[:one_image_path.rindex(".")])
procs = 0
# read image file and crop and get AU mask
pool = mp.Pool(processes=proc_num)
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
for idx, line in enumerate(au_file_obj):
if idx == 0:
continue
lines = line.split(",")
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
pool.apply_async(func=delegate_mask_crop,
args=(img_path, True, queue))
procs += 1
# p = mp.Process(target=delegate_mask_crop, args=(img_path, True, queue))
# procs.append(p)
# p.start()
for i in range(procs):
try:
entry = queue.get(block=True, timeout=360)
resultdict[entry[0]] = (entry[1], entry[2])
except Exception:
print("queue block time out")
break
pool.close()
pool.join()
del pool
else: # only one process
one_image_path = os.listdir(config.RGB_PATH["BP4D"] + os.sep +
subject_name + os.sep +
sequence_name)[0]
zfill_len = len(one_image_path[:one_image_path.rindex(".")])
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
for idx, line in enumerate(au_file_obj):
lines = line.split(",")
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=True)
# note that above AU_mask_dict, each AU may have mask that contains multiple separate regions
resultdict[img_path] = (cropped_face, AU_mask_dict)
print("one image :{} done".format(img_path))
except IndexError:
print("img_path:{} cannot obtain 68 landmark".format(
img_path))
pass
# for p in procs:
# p.join()
AU_column_idx = {}
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
video_info = []
for idx, line in enumerate(
au_file_obj): # each line represent a frame image
line = line.rstrip()
lines = line.split(",")
if idx == 0: # header define which column is which Action Unit
for col_idx, AU in enumerate(lines[1:]):
AU_column_idx[AU] = col_idx + 1 # read header
continue # read head over , continue
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
if img_path not in resultdict:
print("img_path:{} landmark not found, continue".format(
img_path))
continue
cropped_face, AU_mask_dict = resultdict[img_path]
all_couple_mask_dict = OrderedDict()
for AU in sorted(map(
int, config.AU_ROI.keys())): # ensure same order
all_couple_mask_dict[au_couple_dict[str(
AU)]] = AU_mask_dict[str(AU)]
au_label_dict = {
AU: int(lines[AU_column_idx[AU]])
for AU in config.AU_ROI.keys()
} # store real AU label
if cut and all(_au_label == 0
for _au_label in au_label_dict.values()):
continue
all_labels = list() # 开始拼接all_labels
for AU_couple in all_couple_mask_dict.keys(
): # 顺序与all_couple_mask_dict一致
child_AU_couple_list = au_couple_child_dict[AU_couple]
AU_couple = set(AU_couple)
for child_AU_couple in child_AU_couple_list:
AU_couple.update(
child_AU_couple
) # label fetch: combine child region's AU
if not is_binary_AU: # in CRF, CRF模式需要将同一个区域的多个AU用逗号分隔，拼接
concat_AU = []
for AU in AU_couple:
if au_label_dict[AU] == 1:
concat_AU.append(AU)
elif au_label_dict[AU] == 9:
concat_AU.append("?{}".format(AU))
if len(concat_AU) == 0:
all_labels.append(
"0") # 若该区域压根没有任何AU出现，为了让只支持单label的CRF工作，用0来代替
else:
all_labels.append(",".join(concat_AU))
else: # convert to np.array which is AU_bin
AU_bin = np.zeros(len(config.AU_SQUEEZE)).astype(
np.uint8)
for AU in AU_couple:
if au_label_dict[AU] == 9:
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], -1)
elif au_label_dict[AU] == 1:
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], 1)
all_labels.append(tuple(AU_bin))
video_info.append({
"frame":
frame,
"cropped_face":
cropped_face,
"all_couple_mask_dict":
all_couple_mask_dict,
"all_labels":
all_labels,
"video_id":
subject_name + "_" + sequence_name
})
resultdict.clear()
if video_info:
yield video_info, subject_name
else:
print("error video_info:{}".format(file_name))
def get_from_entry(self, img_path, AU_set, database_name):
if not os.path.exists(img_path):
raise IndexError("image file_path: {} not exist!".format(img_path))
try:
# print("begin fetch cropped image and bbox {}".format(img_path))
key_prefix = self.database + "|"
if self.pretrained_target is not None and len(
self.pretrained_target) > 0:
key_prefix = self.pretrained_target + "|"
rgb_img_path = config.RGB_PATH[
self.database] + os.path.sep + "/".join(
img_path.split("/")[-3:])
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
img_path,
rgb_img_path,
channel_first=True,
mc_manager=self.mc_manager,
key_prefix=key_prefix)
except IndexError:
# print("read image error:{}".format(read_img_path))
# return AUDataset.get_example(self, i-1) # 不得已为之
raise IndexError(
"fetch crooped face and mask error:{} ! face landmark may not found."
.format(img_path))
non_AU_set = set()
for AU in config.AU_ROI.keys():
if AU not in AU_set and "?{}".format(AU) not in AU_set:
non_AU_set.add("-{}".format(AU))
unknown_AU_set = set()
known_AU_set = set()
for AU in AU_set:
if AU.startswith("?"):
unknown_AU_set.add(AU)
else:
known_AU_set.add(AU)
all_AU_set = set()
all_AU_set.update(non_AU_set)
all_AU_set.update(unknown_AU_set)
all_AU_set.update(known_AU_set)
current_AU_couple = defaultdict(
set) # key = AU couple, value = AU 用于合并同一个区域的不同AU
couple_box_dict = OrderedDict() # key= AU couple
# mask_path_dict's key AU maybe 3 or -2 or ?5
for AU in all_AU_set:
_AU = AU if AU.isdigit() else AU[1:]
# print("AU:",AU,"_AU:",_AU)
try:
current_AU_couple[self.au_couple_dict[_AU]].add(
AU
) # value list may contain ?2 or -1, 所以这一步会把脸上有的，没有的AU都加上
except KeyError:
print(list(self.au_couple_dict.keys()), _AU)
raise
for AU, box_list in sorted(AU_box_dict.items(),
key=lambda e: int(e[0])):
_AU = AU if AU.isdigit() else AU[1:]
if _AU in config.SYMMETRIC_AU and len(box_list) == 1:
box_list.append(random.choice(box_list))
couple_box_dict[
self.au_couple_dict[_AU]] = box_list # 所以这一步会把脸上有的，没有的AU都加上
label = [
] # one box may have multiple labels. so each entry is 10101110 binary code
bbox = [] # AU = 0背景的box是随机取的
self.assign_label(couple_box_dict, current_AU_couple, bbox, label)
# print("assigned label over")
assert len(bbox) > 0
bbox = np.stack(bbox).astype(np.float32)
label = np.stack(label).astype(np.int32)
# bbox, label = self.proposal(bbox, label) # 必须保证每个batch拿到的box数量一样
assert bbox.shape[0] == label.shape[0]
return cropped_face, bbox, label
def parallel_landmark_and_conn_component(img_path, landmark_dict, AU_box_dict):
orig_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
if landmark_dict is None or len(landmark_dict) == 0:
try:
landmark_dict, _, _ = FaceMaskCropper.landmark.landmark(
image=orig_img, need_txt_img=False) # slow
except IndexError:
if AU_box_dict is None:
AU_box_dict = defaultdict(list)
for AU in config.AU_ROI.keys():
if AU in config.SYMMETRIC_AU:
for _ in range(2):
AU_box_dict[AU].append(
(0.0, 0.0, config.IMG_SIZE[1],
config.IMG_SIZE[0]))
else:
AU_box_dict[AU].append(
(0.0, 0.0, config.IMG_SIZE[1], config.IMG_SIZE[0]))
return img_path, AU_box_dict, None, True
cropped_face, rect = FaceMaskCropper.dlib_face_crop(
orig_img, landmark_dict)
cropped_face = cv2.resize(cropped_face, config.IMG_SIZE)
del orig_img
if AU_box_dict is None:
AU_box_dict = defaultdict(list)
for AU in config.AU_ROI.keys():
mask = crop_face_mask_from_landmark(
AU,
landmark_dict,
cropped_face,
rect,
landmarker=FaceMaskCropper.landmark)
connect_arr = cv2.connectedComponents(
mask, connectivity=8,
ltype=cv2.CV_32S) # mask shape = 1 x H x W
component_num = connect_arr[0]
label_matrix = connect_arr[1]
# convert mask polygon to rectangle
for component_label in range(1, component_num):
row_col = list(zip(*np.where(label_matrix == component_label)))
row_col = np.array(row_col)
y_min_index = np.argmin(row_col[:, 0])
y_min = row_col[y_min_index, 0]
x_min_index = np.argmin(row_col[:, 1])
x_min = row_col[x_min_index, 1]
y_max_index = np.argmax(row_col[:, 0])
y_max = row_col[y_max_index, 0]
x_max_index = np.argmax(row_col[:, 1])
x_max = row_col[x_max_index, 1]
# same region may be shared by different AU, we must deal with it
coordinates = (y_min, x_min, y_max, x_max)
if y_min == y_max and x_min == x_max: # 尖角处会产生孤立的单个点，会不会有一个mask只有尖角？
# print(("single point mask: img:{0} mask:{1}".format(self._images[i], mask_path)))
# 然后用concat_example来拼接起来
continue
AU_box_dict[AU].append(coordinates)
del label_matrix
del mask
for AU, box_lst in AU_box_dict.items():
AU_box_dict[AU] = sorted(box_lst, key=lambda e: int(e[3]))
return img_path, AU_box_dict, landmark_dict, False
def get_example(self, i):
'''
Returns a color image and bounding boxes. The image is in CHW format.
The returned image is RGB.
:param i: the index of the example
:return: tuple of an image and its all bounding box
'''
if i > len(self.result_data):
raise IndexError("Index too large")
rgb_path, _, AU_set, database = self.result_data[i]
if self.get_npz_name(
self.npz_dir, database, self.fold, self.split_index,
self.extract_sequence_key(rgb_path).replace("/", "_")):
return None, None, None, None, rgb_path
flow_path_list = self.collect_flow_image_paths(i)
key_prefix = self.database + "|"
if self.pretrained_target is not None and len(
self.pretrained_target) > 0:
key_prefix = self.pretrained_target + "|"
flow_face_list = []
for flow_dict in flow_path_list:
adjacent_rgb_path = flow_dict["rgb"]
adjacent_flow_path = flow_dict["flow"]
try:
# FIXME read too slow, use the same rgb path to accelerate speed . but this trick is not accurate
flow_face, _ = FaceMaskCropper.get_cropface_and_box(
adjacent_flow_path,
adjacent_rgb_path,
channel_first=True,
mc_manager=self.mc_manager,
key_prefix=key_prefix)
flow_face = flow_face[:2, :, :] # 2, H, W
flow_face_list.append(
flow_face
) # only use two channel x and y of optical flow image
except IndexError:
print("image path : {} not get box".format(adjacent_rgb_path))
flow_face = cv2.imread(adjacent_flow_path)
flow_face = cv2.resize(flow_face, config.IMG_SIZE)
flow_face = np.transpose(flow_face, axes=(2, 0, 1))
flow_face = flow_face[:2, :, :]
flow_face_list.append(flow_face)
flow_face_list = np.stack(flow_face_list) # T, C, H, W
if len(flow_face_list) < self.L:
rest_pad_len = self.L - len(flow_face_list)
flow_face_list = np.pad(flow_face_list, ((0, rest_pad_len), (0, 0),
(0, 0), (0, 0)), 'mean')
assert flow_face_list.shape[0] == self.L
try:
# print("begin fetch cropped image and bbox {}".format(img_path))
rgb_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
rgb_path,
rgb_path,
channel_first=True,
mc_manager=self.mc_manager,
key_prefix=key_prefix)
except IndexError:
print("image path : {} not get box".format(rgb_path))
label = np.zeros(len(config.AU_SQUEEZE), dtype=np.int32)
for AU in AU_set:
np.put(label, config.AU_SQUEEZE.inv[AU], 1)
if self.paper_report_label_idx:
label = label[self.paper_report_label_idx]
rgb_face = np.transpose(
cv2.resize(cv2.imread(rgb_path), config.IMG_SIZE), (2, 0, 1))
whole_bbox = np.tile(
np.array(
[1, 1, config.IMG_SIZE[1] - 1, config.IMG_SIZE[0] - 1],
dtype=np.float32), (config.BOX_NUM[database], 1))
whole_label = np.tile(label, (config.BOX_NUM[database], 1))
return rgb_face, flow_face_list, whole_bbox, whole_label, rgb_path
current_AU_couple = defaultdict(
set) # key = AU couple, value = AU 用于合并同一个区域的不同AU
couple_box_dict = OrderedDict() # key= AU couple
# mask_path_dict's key AU maybe 3 or -2 or ?5
for AU in AU_set:
assert AU.isdigit()
try:
current_AU_couple[self.au_couple_dict[AU]].add(
AU
) # value list may contain ?2 or -1, 所以这一步会把脸上有的，没有的AU都加上
except KeyError:
print(list(self.au_couple_dict.keys()), AU)
raise
for AU, box_list in sorted(AU_box_dict.items(),
key=lambda e: int(e[0])):
assert AU.isdigit()
if AU in config.SYMMETRIC_AU and len(box_list) == 1:
box_list.append(random.choice(box_list))
couple_box_dict[
self.au_couple_dict[AU]] = box_list # 所以这一步会把脸上有的，没有的AU都加上
label = [
] # one box may have multiple labels. so each entry is 10101110 binary code
bbox = [] # AU = 0背景的box是随机取的
self.assign_label(couple_box_dict, current_AU_couple, bbox, label)
# print("assigned label over")
assert len(bbox) > 0
bbox = np.stack(bbox).astype(np.float32)
label = np.stack(label).astype(np.int32)
# bbox, label = self.proposal(bbox, label) # 必须保证每个batch拿到的box数量一样
assert bbox.shape[0] == label.shape[0]
if self.paper_report_label_idx is not None:
label = label[:, self.paper_report_label_idx]
return rgb_face, flow_face_list, bbox, label, rgb_path
def get_example(self, i):
'''
Returns a color image and bounding boxes. The image is in CHW format.
The returned image is RGB.
:param i: the index of the example
:return: tuple of an image and its all bounding box
'''
if i > len(self.result_data):
raise IndexError("Index too large")
img_path, from_img_path, AU_set, database_name = self.result_data[i]
if not os.path.exists(img_path):
raise IndexError("image file_path: {} not exist!".format(img_path))
try:
# print("begin fetch cropped image and bbox {}".format(img_path))
read_img_path = img_path if from_img_path == "#" else from_img_path
rgb_img_path = config.RGB_PATH[
self.database] + os.path.sep + os.path.sep.join(
read_img_path.split("/")[-3:])
img_id = "/".join(
(read_img_path.split("/")[-3], read_img_path.split("/")[-2],
read_img_path.split("/")[-1]
[:read_img_path.split("/")[-1].index(".")]))
key_prefix = self.database + "@{}".format(
self.img_resolution) + "|"
if self.pretrained_target is not None and len(
self.pretrained_target) > 0:
key_prefix = self.pretrained_target + "|"
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
read_img_path,
rgb_img_path,
channel_first=True,
mc_manager=self.mc_manager,
key_prefix=key_prefix)
except IndexError:
print("crop image error:{}".format(read_img_path))
label = np.zeros(len(config.AU_SQUEEZE), dtype=np.int32)
for AU in AU_set:
np.put(label, config.AU_SQUEEZE.inv[AU], 1)
face = np.transpose(
cv2.resize(cv2.imread(read_img_path), config.IMG_SIZE),
(2, 0, 1))
whole_bbox = np.tile(
np.array(
[1, 1, config.IMG_SIZE[1] - 1, config.IMG_SIZE[0] - 1],
dtype=np.float32), (config.BOX_NUM[database_name], 1))
whole_label = np.tile(label, (config.BOX_NUM[database_name], 1))
return face, whole_bbox, whole_label
# return self.get_example(random.randint(0, len(self)-1)) # 不得已为之
# raise IndexError("fetch crooped face and mask error:{} ! face landmark may not found.".format(read_img_path))
# print("fetch over")
non_AU_set = set()
for AU in config.AU_ROI.keys():
if AU not in AU_set and "?{}".format(AU) not in AU_set:
non_AU_set.add("-{}".format(AU))
unknown_AU_set = set()
known_AU_set = set()
for AU in AU_set:
if AU.startswith("?"):
unknown_AU_set.add(AU)
else:
known_AU_set.add(AU)
all_AU_set = set()
all_AU_set.update(non_AU_set)
all_AU_set.update(unknown_AU_set)
all_AU_set.update(known_AU_set)
current_AU_couple = defaultdict(
set) # key = AU couple, value = AU 用于合并同一个区域的不同AU
couple_box_dict = OrderedDict() # key= AU couple
# mask_path_dict's key AU maybe 3 or -2 or ?5
for AU in all_AU_set:
_AU = AU if AU.isdigit() else AU[1:]
# print("AU:",AU,"_AU:",_AU)
try:
current_AU_couple[self.au_couple_dict[_AU]].add(
AU
) # value list may contain ?2 or -1, 所以这一步会把脸上有的，没有的AU都加上
except KeyError:
print(list(self.au_couple_dict.keys()), _AU)
raise
for AU, box_list in sorted(AU_box_dict.items(),
key=lambda e: int(e[0])):
_AU = AU if AU.isdigit() else AU[1:]
couple_box_dict[
self.au_couple_dict[_AU]] = box_list # 所以这一步会把脸上有的，没有的AU都加上
label = [
] # one box may have multiple labels. so each entry is 10101110 binary code
bbox = [] # AU = 0背景的box是随机取的
AU_couple_lst = []
self.assign_label(couple_box_dict, current_AU_couple, bbox, label,
AU_couple_lst)
# print("assigned label over")
if len(bbox) == 0:
print("no box found on face")
return self.get_example(i - 1)
bbox = np.stack(bbox).astype(np.float32)
label = np.stack(label).astype(np.int32)
# bbox, label = self.proposal(bbox, label) # 必须保证每个batch拿到的box数量一样
assert bbox.shape[0] == label.shape[0]
return cropped_face, bbox, label # AU_couple_lst为了random shift bbox
def get_parallel_example(self, fetch_list):
parallel_data = []
img_path_label_dict = dict()
for i in fetch_list:
if i > len(self.result_data):
raise IndexError("Index too large")
img_path, AU_set, database_name = self.result_data[i]
img_path_label_dict[img_path] = AU_set
# print("begin fetch cropped image and bbox {}".format(img_path))
key_prefix = self.database + "|"
key = key_prefix+ "/".join((img_path.split("/")[-3], img_path.split("/")[-2],img_path.split("/")[-1]))
landmark_dict = None
AU_box_dict = None
if self.mc_manager is not None and key in self.mc_manager:
result = self.mc_manager.get(key)
landmark_dict = result.get("landmark_dict",None)
AU_box_dict = result.get("AU_box_dict", None)
parallel_data.append((img_path, landmark_dict, AU_box_dict))
with Pool(processes=3) as pool:
parallel_result = pool.starmap_async(parallel_landmark_and_conn_component, parallel_data)
img_dict = dict()
for img_path, *_ in parallel_data:
img_dict[img_path] = cv2.imread(img_path, cv2.IMREAD_COLOR)
parallel_result.wait()
# pool.close()
# pool.join()
for img_path, AU_box_dict, landmark_dict, box_is_whole_image in parallel_result.get():
cropped_face = img_dict[img_path]
rect = None
if landmark_dict is not None:
cropped_face, rect = FaceMaskCropper.dlib_face_crop(img_dict[img_path], landmark_dict)
cropped_face = cv2.resize(cropped_face, config.IMG_SIZE)
cropped_face = np.transpose(cropped_face, (2, 0, 1)) # put channel first!
AU_set = img_path_label_dict[img_path]
key_prefix = self.database + "|"
key = key_prefix + "/".join((img_path.split("/")[-3], img_path.split("/")[-2], img_path.split("/")[-1]))
if self.mc_manager is not None:
save_dict = {"landmark_dict": landmark_dict, "AU_box_dict": AU_box_dict, "crop_rect":rect}
self.mc_manager.set(key, save_dict)
AU_couple_gt_label = defaultdict(set) # key = AU couple, value = AU 用于合并同一个区域的不同AU
couple_box_dict = DefaultOrderedDict(list) # key= AU couple
# mask_path_dict's key AU maybe 3 or -2 or ?5
if box_is_whole_image:
for AU in config.AU_SQUEEZE.values():
AU_couple_gt_label[self.au_couple_dict[AU]] = AU_set
else:
for AU in config.AU_SQUEEZE.values():
if AU in AU_set:
AU_couple_gt_label[self.au_couple_dict[AU]].add(AU)
for AU, box_list in sorted(AU_box_dict.items(), key=lambda e: int(e[0])):
assert AU.isdigit()
couple_box_dict[self.au_couple_dict[AU]] = box_list # couple_box_dict will contain all AU including not occur on face
label = [] # one box may have multiple labels. so each entry is 10101110 binary code
bbox = []
self.assign_label(couple_box_dict, AU_couple_gt_label, bbox, label)
assert len(bbox) > 0
# print("assigned label over")
bbox = np.stack(bbox).astype(np.float32)
label = np.stack(label).astype(np.int32)
assert bbox.shape[0] == label.shape[0]
yield cropped_face, bbox, label