def delegate_mask_crop(img_path, channal_first, queue):
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channal_first)
print("crop {} done".format(img_path))
queue.put((img_path, cropped_face, AU_mask_dict), block=True)
except IndexError:
pass
def generate_AUCouple_ROI_mask_image(database_name, img_path):
adaptive_AU_database(database_name)
global MASK_COLOR
mask_color_lst = []
for color in MASK_COLOR:
mask_color_lst.append(color_bgr(color))
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True,
color_bgr(RED),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr, axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
idx = 0
gen_face_lst = dict()
AU_couple_mask = dict()
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in already_fill_AU:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
AU_couple_mask[AU_couple] = mask
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = random.choice(mask_color_lst)
idx += 1
new_face = cv2.addWeighted(cropped_face, 0.75, color_mask, 0.25, 0)
gen_face_lst[AU_couple] = new_face
return gen_face_lst, AU_couple_mask
def generate_AUCouple_ROI_mask_image(self, database_name, img_path,
roi_activate):
adaptive_AU_database(database_name)
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True, (0, 0, 255),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr,
axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
AUCouple_face_dict = dict()
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in already_fill_AU or AU_couple not in roi_activate:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = (199, 21, 133)
new_face = cv2.add(cropped_face, color_mask)
AUCouple_face_dict[AU_couple] = new_face
return AUCouple_face_dict
def read_DISFA_video_label(output_dir,
is_binary_AU,
is_need_adaptive_AU_relation=False,
force_generate=True,
proc_num=10,
cut=False,
train_subject=None):
mgr = mp.Manager()
queue = mgr.Queue(maxsize=20000)
for orientation in ["Left", "Right"]:
if is_need_adaptive_AU_relation:
adaptive_AU_relation(
) # delete AU relation pair occur in same facial region
au_couple_dict = get_zip_ROI_AU()
au_couple_child_dict = get_AU_couple_child(au_couple_dict)
DISFA_base_dir = config.DATA_PATH["DISFA"]
label_file_dir = DISFA_base_dir + "/ActionUnit_Labels/"
img_folder = DISFA_base_dir + "/Img_{}Camera".format(orientation)
for video_name in os.listdir(label_file_dir): # each file is a video
is_train = True if video_name in train_subject else False
if not force_generate:
prefix = "train" if is_train else "test"
target_file_path = output_dir + os.sep + prefix + os.sep + video_name + "_" + orientation + ".npz"
if os.path.exists(target_file_path):
continue
resultdict = {}
if proc_num > 1:
pool = mp.Pool(processes=proc_num)
procs = 0
one_file_name = os.listdir(label_file_dir + os.sep +
video_name)[0]
with open(
label_file_dir + os.sep + video_name + os.sep +
one_file_name, "r") as file_obj:
for idx, line in enumerate(file_obj):
line = line.strip()
if line:
frame = line.split(",")[0]
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
pool.apply_async(func=delegate_mask_crop,
args=(img_path, True, queue))
procs += 1
for i in range(procs):
try:
entry = queue.get(block=True, timeout=60)
resultdict[entry[0]] = (entry[1], entry[2])
except Exception:
print("queue block time out")
break
pool.close()
pool.join()
del pool
else: # only one process
one_file_name = os.listdir(label_file_dir + os.sep +
video_name)[0]
with open(
label_file_dir + os.sep + video_name + os.sep +
one_file_name, "r") as file_obj:
for idx, line in enumerate(file_obj):
line = line.strip()
if line:
frame = line.split(",")[0]
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, True)
resultdict[img_path] = (cropped_face,
AU_mask_dict)
except IndexError:
pass
frame_label = dict()
video_info = []
video_img_path_set = set()
for file_name in os.listdir(
label_file_dir + os.sep +
video_name): # each file is one AU ( video file )
AU = file_name[file_name.index("au") + 2:file_name.rindex(".")]
with open(
label_file_dir + os.sep + video_name + os.sep +
file_name, "r") as file_obj:
for line in file_obj:
frame = int(line.split(",")[0])
AU_intensity = int(line.split(",")[1])
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
video_img_path_set.add((frame, img_path))
if frame not in frame_label:
frame_label[frame] = set()
if AU_intensity >= 1: # NOTE that we apply AU_intensity >= 1
frame_label[int(frame)].add(AU) #存储str类型
for frame, img_path in sorted(video_img_path_set,
key=lambda e: int(e[0])):
if img_path not in resultdict:
continue
AU_set = frame_label[frame] # it is whole image's AU set
if cut and len(AU_set) == 0:
continue
cropped_face, AU_mask_dict = resultdict[img_path]
all_couple_mask_dict = OrderedDict()
for AU in sorted(map(
int, config.AU_ROI.keys())): # ensure same order
all_couple_mask_dict[au_couple_dict[str(
AU)]] = AU_mask_dict[str(AU)]
all_labels = list() # 开始拼接all_labels
for AU_couple in all_couple_mask_dict.keys(
): # 顺序与all_couple_mask_dict一致
child_AU_couple_list = au_couple_child_dict[AU_couple]
AU_couple = set(AU_couple)
for child_AU_couple in child_AU_couple_list:
AU_couple.update(
child_AU_couple) # combine child region's AU
if not is_binary_AU: # in CRF, CRF模式需要将同一个区域的多个AU用逗号分隔，拼接
concat_AU = []
for AU in AU_couple:
if AU in AU_set: # AU_set 存储真实AU(ground truth label):str类型
concat_AU.append(AU)
if len(concat_AU) == 0:
all_labels.append(
"0") # 若该区域压根没有任何AU出现，为了让只支持单label的CRF工作，用0来代替
else:
all_labels.append(",".join(sorted(concat_AU)))
else: # convert to np.array which is AU_bin
AU_bin = np.zeros(len(config.AU_SQUEEZE)).astype(
np.uint8)
for AU in AU_couple:
if AU in AU_set: # judge if this region contain which subset of whole image's AU_set
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], 1)
all_labels.append(tuple(AU_bin))
video_info.append({
"frame": frame,
"cropped_face": cropped_face,
"all_couple_mask_dict": all_couple_mask_dict,
"all_labels": all_labels,
"video_id": video_name + "_" + orientation
})
resultdict.clear()
if video_info:
yield video_info, video_name
else:
print(
"error in file:{} no video found".format(video_name + "_" +
orientation))
def read_BP4D_video_label(output_dir,
is_binary_AU,
is_need_adaptive_AU_relation=False,
force_generate=True,
proc_num=10,
cut=False,
train_subject=None):
'''
:param
output_dir : 用于检查如果目标的文件已经存在，那么就不再生成
is_binary_AU:
True --> return AU_binary 01010100
False --> used for CRF mode: single true AU label CRF/ or AU combination separate by comma
:yield: 每个视频video收集齐了yield回去，视频中每一帧返回3部分：
1. "img_path": /path/to/image
1."all_couple_mask_dict": 是OrderedDict，包含所有区域的mask，不管AU是不是+1，还是-1(不管AU出现没出现)，key是AU_couple，来自于au_couple_dict = get_zip_ROI_AU()
2."labels": 是list，index与all_couple_mask_dict一致，其中每个label
要么是binary形式01010110，
要么是3,4（由于一块位置可以发生多个AU，因此可以用逗号隔开的字符串来返回），根据is_binary_AU返回不同的值
'''
mgr = mp.Manager()
queue = mgr.Queue(maxsize=20000)
if is_need_adaptive_AU_relation:
adaptive_AU_relation(
) # delete AU relation pair occur in same facial region
au_couple_dict = get_zip_ROI_AU()
au_couple_child_dict = get_AU_couple_child(
au_couple_dict) # AU_couple => list of child AU_couple
# if need_translate_combine_AU ==> "mask_path_dict":{(2,3,4): /pathtomask} convert to "mask_path_dict":{110: /pathtomask}
# each is dict : {"img": /path/to/img, "mask_path_dict":{(2,3,4): /pathtomask}, }
BP4D_base_dir_path = config.DATA_PATH["BP4D"]
label_file_dir = BP4D_base_dir_path + "/AUCoding/"
for file_name in os.listdir(label_file_dir): # each file is a video
subject_name = file_name[:file_name.index("_")]
sequence_name = file_name[file_name.index("_") +
1:file_name.rindex(".")]
is_train = True if subject_name in train_subject else False
if not force_generate:
prefix = "train" if is_train else "test"
target_file_path = output_dir + os.sep + prefix + os.sep + subject_name + "_" + sequence_name + ".npz"
if os.path.exists(target_file_path):
continue
resultdict = {}
if proc_num > 1:
one_image_path = os.listdir(config.RGB_PATH["BP4D"] + os.sep +
subject_name + os.sep +
sequence_name)[0]
zfill_len = len(one_image_path[:one_image_path.rindex(".")])
procs = 0
# read image file and crop and get AU mask
pool = mp.Pool(processes=proc_num)
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
for idx, line in enumerate(au_file_obj):
if idx == 0:
continue
lines = line.split(",")
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
pool.apply_async(func=delegate_mask_crop,
args=(img_path, True, queue))
procs += 1
# p = mp.Process(target=delegate_mask_crop, args=(img_path, True, queue))
# procs.append(p)
# p.start()
for i in range(procs):
try:
entry = queue.get(block=True, timeout=360)
resultdict[entry[0]] = (entry[1], entry[2])
except Exception:
print("queue block time out")
break
pool.close()
pool.join()
del pool
else: # only one process
one_image_path = os.listdir(config.RGB_PATH["BP4D"] + os.sep +
subject_name + os.sep +
sequence_name)[0]
zfill_len = len(one_image_path[:one_image_path.rindex(".")])
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
for idx, line in enumerate(au_file_obj):
lines = line.split(",")
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=True)
# note that above AU_mask_dict, each AU may have mask that contains multiple separate regions
resultdict[img_path] = (cropped_face, AU_mask_dict)
print("one image :{} done".format(img_path))
except IndexError:
print("img_path:{} cannot obtain 68 landmark".format(
img_path))
pass
# for p in procs:
# p.join()
AU_column_idx = {}
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
video_info = []
for idx, line in enumerate(
au_file_obj): # each line represent a frame image
line = line.rstrip()
lines = line.split(",")
if idx == 0: # header define which column is which Action Unit
for col_idx, AU in enumerate(lines[1:]):
AU_column_idx[AU] = col_idx + 1 # read header
continue # read head over , continue
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
if img_path not in resultdict:
print("img_path:{} landmark not found, continue".format(
img_path))
continue
cropped_face, AU_mask_dict = resultdict[img_path]
all_couple_mask_dict = OrderedDict()
for AU in sorted(map(
int, config.AU_ROI.keys())): # ensure same order
all_couple_mask_dict[au_couple_dict[str(
AU)]] = AU_mask_dict[str(AU)]
au_label_dict = {
AU: int(lines[AU_column_idx[AU]])
for AU in config.AU_ROI.keys()
} # store real AU label
if cut and all(_au_label == 0
for _au_label in au_label_dict.values()):
continue
all_labels = list() # 开始拼接all_labels
for AU_couple in all_couple_mask_dict.keys(
): # 顺序与all_couple_mask_dict一致
child_AU_couple_list = au_couple_child_dict[AU_couple]
AU_couple = set(AU_couple)
for child_AU_couple in child_AU_couple_list:
AU_couple.update(
child_AU_couple
) # label fetch: combine child region's AU
if not is_binary_AU: # in CRF, CRF模式需要将同一个区域的多个AU用逗号分隔，拼接
concat_AU = []
for AU in AU_couple:
if au_label_dict[AU] == 1:
concat_AU.append(AU)
elif au_label_dict[AU] == 9:
concat_AU.append("?{}".format(AU))
if len(concat_AU) == 0:
all_labels.append(
"0") # 若该区域压根没有任何AU出现，为了让只支持单label的CRF工作，用0来代替
else:
all_labels.append(",".join(concat_AU))
else: # convert to np.array which is AU_bin
AU_bin = np.zeros(len(config.AU_SQUEEZE)).astype(
np.uint8)
for AU in AU_couple:
if au_label_dict[AU] == 9:
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], -1)
elif au_label_dict[AU] == 1:
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], 1)
all_labels.append(tuple(AU_bin))
video_info.append({
"frame":
frame,
"cropped_face":
cropped_face,
"all_couple_mask_dict":
all_couple_mask_dict,
"all_labels":
all_labels,
"video_id":
subject_name + "_" + sequence_name
})
resultdict.clear()
if video_info:
yield video_info, subject_name
else:
print("error video_info:{}".format(file_name))
def generate_mask_contain_img(database_name, img_path):
adaptive_AU_database(database_name)
mask_color = {}
for parent_color, child_color in MASK_CONTAIN.items():
mask_color[color_bgr(parent_color)] = color_bgr(child_color)
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
AU_couple_child = get_AU_couple_child(AU_couple_dict)
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True,
color_bgr(RED),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr, axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
gen_face_lst = dict()
all_child_set = set()
for child_set in AU_couple_child.values():
for child in child_set:
all_child_set.add(child)
new_face = np.zeros_like(cropped_face)
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in all_child_set:
continue
if AU_couple in already_fill_AU:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
child_AU_set = AU_couple_child[AU_couple]
color_parent = list(MASK_CONTAIN.keys())[0]
color_child = MASK_CONTAIN[color_parent]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = color_parent
# cv2.addWeighted(color_mask,0.5, color_mask,1-0.5,0,color_mask)
if np.any(new_face):
cropped_face = new_face
cv2.addWeighted(cropped_face, 1, color_mask, 0.3, 0, new_face, -1)
for child_AU in child_AU_set:
if child_AU in already_fill_AU:
continue
already_fill_AU.add(child_AU)
mask = AU_mask_dict[child_AU[0]]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = random.choice(color_child)
cv2.addWeighted(new_face, 1, color_mask, 0.5, 0, new_face, -1)
return new_face