def regenerate_miss_mask(database, mp_num, miss_file_path):
pool = mp.Pool()
AU_couple_dict = get_zip_ROI_AU()
orig_file_path_lst = set()
landmark = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
with open(miss_file_path, 'r') as file_obj:
for line in file_obj:
miss_mask_path = line.strip()
subject_name = miss_mask_path.split("/")[-3]
sequenece_name = miss_mask_path.split("/")[-2]
frame = miss_mask_path.split("/")[-1]
frame = frame[:frame.index("_")]
orig_train_file_path = config.DATA_PATH[
"BP4D"] + "/release/BP4D-training/{0}/{1}/{2}.jpg".format(
subject_name, sequenece_name, frame)
orig_file_path_lst.add(orig_train_file_path)
split_list = lambda A, n=3: [A[i:i + n] for i in range(0, len(A), n)]
sub_list = split_list(list(orig_file_path_lst),
len(orig_file_path_lst) // mp_num)
for sub in sub_list:
pool.apply_async(sub_process_miss_mask,
args=(database, landmark, sub, AU_couple_dict))
pool.close()
pool.join()
def check_box_and_cropface(orig_img_path, channel_first=False):
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_box(
orig_img_path, channel_first=channel_first, mc_manager=None)
AU_couple = get_zip_ROI_AU()
already_couple = set()
# cropped_face = np.transpose(cropped_face, (2, 0, 1))
# cropped_face, params = transforms.random_flip(
# cropped_face, x_random=True, return_param=True)
# cropped_face = np.transpose(cropped_face, (1, 2, 0))
i = 0
for AU, box_ls in AU_mask_dict.items():
current_AU_couple = AU_couple[AU]
if current_AU_couple in already_couple:
continue
already_couple.add(current_AU_couple)
for box in box_ls:
box = np.asarray([box])
box = transforms.flip_bbox(box, (512, 512), x_flip=False)
x_min, y_min = box[0][1], box[0][0]
x_max, y_max = box[0][3], box[0][2]
print(box)
cp_croped = cropped_face.copy()
cv2.rectangle(cp_croped, (x_min, y_min), (x_max, y_max),
(0, 255, 0), 1)
cv2.imwrite(
"/home2/mac/test1/AU_{0}_{1}.png".format(
",".join(current_AU_couple), i), cp_croped)
i += 1
print(i)
def adaptive_AU_relation(database_name):
'''
must called after adaptive_AU_database
从config.AU_RELATION_BP4D中删去同一个区域的AU
比如说config.AU_RELATION_BP4D 中有(10, 12) 但10和12已经是同一个区域了，因此删掉这种关系。
另外需要注意另一种情况：再比如说config.AU_RELATION_BP4D 中有(10, 13)，但10和12再字典中组合成新的区域了，而13也和另一个AU组合成了新的区域。
'''
new_AU_relation = list()
AU_couple = get_zip_ROI_AU()
already_same_region_set = set()
for AU, couple_tuple in AU_couple.items():
for AU_a, AU_b in itertools.combinations(couple_tuple, 2):
already_same_region_set.add(tuple(sorted([int(AU_a), int(AU_b)])))
if database_name == "BP4D":
for AU_tuple in config.AU_RELATION_BP4D:
if tuple(sorted(AU_tuple)) not in already_same_region_set:
new_AU_relation.append(tuple(sorted(AU_tuple)))
config.AU_RELATION_BP4D = new_AU_relation
elif database_name == "DISFA":
for AU_tuple in config.AU_RELATION_DISFA:
if tuple(sorted(AU_tuple)) not in already_same_region_set:
new_AU_relation.append(tuple(sorted(AU_tuple)))
config.AU_RELATION_DISFA = new_AU_relation
def __init__(self,
database,
L,
fold,
split_name,
split_index,
mc_manager,
train_all_data,
two_stream_mode,
pretrained_target="",
paper_report_label_idx=None):
self.database = database
self.split_name = split_name
self.L = L # used for the optical flow image fetch at before L/2 and after L/2
self.au_couple_dict = get_zip_ROI_AU()
self.mc_manager = mc_manager
self.au_couple_child_dict = get_AU_couple_child(self.au_couple_dict)
self.AU_intensity_label = {
} # subject + "/" + emotion_seq + "/" + frame => ... not implemented
self.pretrained_target = pretrained_target
self.two_stream_mode = two_stream_mode
self.dir = config.DATA_PATH[database] # BP4D/DISFA/ BP4D_DISFA
self.paper_report_label_idx = paper_report_label_idx
if train_all_data:
id_list_file_path = os.path.join(
self.dir + "/idx/{}_fold".format(fold), "full_pretrain.txt")
else:
id_list_file_path = os.path.join(
self.dir + "/idx/{0}_fold".format(fold),
"id_{0}_{1}.txt".format(split_name, split_index))
self.result_data = []
print("idfile:{}".format(id_list_file_path))
with open(id_list_file_path, "r") as file_obj:
for idx, line in enumerate(file_obj):
if line.rstrip():
line = line.rstrip()
relative_path, au_set_str, _, current_database_name = line.split(
"\t")
AU_set = set(
AU for AU in au_set_str.split(',')
if AU in config.AU_ROI and AU in config.AU_SQUEEZE.inv)
if au_set_str == "0":
AU_set = set()
rgb_path = config.RGB_PATH[
current_database_name] + os.path.sep + relative_path # id file 是相对路径
flow_path = config.FLOW_PATH[
current_database_name] + os.path.sep + relative_path
if os.path.exists(rgb_path):
self.result_data.append((rgb_path, flow_path, AU_set,
current_database_name))
self.result_data.sort(key=lambda entry: (
entry[0].split("/")[-3], entry[0].split("/")[-2],
int(entry[0].split("/")[-1][:entry[0].split("/")[-1].rindex(".")]))
)
self._num_examples = len(self.result_data)
print("read id file done, all examples:{}".format(self._num_examples))
def __init__(self,
img_resolution,
database,
fold,
split_name,
split_index,
mc_manager,
prefix="",
pretrained_target=""):
self.database = database
self.img_resolution = img_resolution
self.split_name = split_name
self.au_couple_dict = get_zip_ROI_AU()
self.mc_manager = mc_manager
self.au_couple_child_dict = get_AU_couple_child(self.au_couple_dict)
self.AU_intensity_label = {
} # subject + "/" + emotion_seq + "/" + frame => ... not implemented
self.pretrained_target = pretrained_target
self.dir = config.DATA_PATH[database] # BP4D/DISFA/ BP4D_DISFA
id_list_file_path = os.path.join(
self.dir + "/idx/{0}_fold{1}".format(fold, prefix),
"intensity_{0}_{1}.txt".format(split_name, split_index))
self.result_data = []
self.video_offset = OrderedDict()
self.video_count = defaultdict(int)
print("idfile:{}".format(id_list_file_path))
with open(id_list_file_path, "r") as file_obj:
for idx, line in enumerate(file_obj):
if line.rstrip():
line = line.rstrip()
img_path, au_set_str, from_img_path, current_database_name = line.split(
"\t")
AU_intensity = np.fromstring(au_set_str,
dtype=np.int32,
sep=',')
from_img_path = img_path if from_img_path == "#" else from_img_path
img_path = config.RGB_PATH[
current_database_name] + os.path.sep + img_path # id file 是相对路径
from_img_path = config.RGB_PATH[
current_database_name] + os.path.sep + from_img_path
video_id = "/".join(
[img_path.split("/")[-3],
img_path.split("/")[-2]])
if video_id not in self.video_offset:
self.video_offset[video_id] = len(self.result_data)
self.video_count[video_id] += 1
if os.path.exists(img_path):
self.result_data.append(
(img_path, from_img_path, AU_intensity,
current_database_name))
self.result_data.sort(key=lambda entry: (
entry[0].split("/")[-3], entry[0].split("/")[-2],
int(entry[0].split("/")[-1][:entry[0].split("/")[-1].rindex(".")]))
)
self._num_examples = len(self.result_data)
print("read id file done, all examples:{}".format(self._num_examples))
def __init__(self,
database,
fold,
split_name,
split_index,
mc_manager,
train_all_data=False,
read_type="rgb",
pretrained_target="",
img_resolution=config.IMG_SIZE[0]):
self.database = database
self.img_resolution = img_resolution
self.split_name = split_name # trainval or test
self.read_type = read_type
self.au_couple_dict = get_zip_ROI_AU()
self.au_couple_child_dict = get_AU_couple_child(self.au_couple_dict)
self.AU_intensity_label = {
} # subject + "/" + emotion_seq + "/" + frame => ... not implemented
self.dir = config.DATA_PATH[database] # BP4D/DISFA/ BP4D_DISFA
self.pretrained_target = pretrained_target
self.mc_manager = mc_manager
if train_all_data:
id_list_file_path = os.path.join(
self.dir + "/idx/{}_fold".format(fold), "full_pretrain.txt")
else:
id_list_file_path = os.path.join(
self.dir + "/idx/{0}_fold".format(fold),
"id_{0}_{1}.txt".format(split_name, split_index))
self.result_data = []
print("idfile:{}".format(id_list_file_path))
with open(id_list_file_path, "r") as file_obj:
for idx, line in enumerate(file_obj):
if line.rstrip():
line = line.rstrip()
img_path, au_set_str, from_img_path, current_database_name = line.split(
"\t")
AU_set = set(AU for AU in au_set_str.split(',')
if AU in config.AU_ROI)
if au_set_str == "0":
AU_set = set()
from_img_path = img_path if from_img_path == "#" else from_img_path
img_path = config.RGB_PATH[
current_database_name] + os.path.sep + img_path # id file 是相对路径
from_img_path = config.RGB_PATH[
current_database_name] + os.path.sep + from_img_path
if os.path.exists(img_path):
self.result_data.append(
(img_path, from_img_path, AU_set,
current_database_name))
self.result_data.sort(key=lambda entry: (
entry[0].split("/")[-3], entry[0].split("/")[-2],
int(entry[0].split("/")[-1][:entry[0].split("/")[-1].rindex(".")]))
)
self._num_examples = len(self.result_data)
print("read id file done, all examples:{}".format(self._num_examples))
def __init__(self, host):
self.AU_couple = get_zip_ROI_AU()
if import_lib:
self.mc = mc.Client([host],
binary=True,
behaviors={
'tcp_nodelay': True,
"ketama": True
})
else:
self.mc = mc.Client([host], debug=0)
def check_BP4D_multi_label(label_file_dir, au_mask_dir):
au_couple_dict = get_zip_ROI_AU()
combine_AU = defaultdict(set)
all_AU = set()
all_co_occur_AU = defaultdict(int)
for file_name in os.listdir(label_file_dir):
subject_name = file_name[:file_name.index("_")]
sequence_name = file_name[file_name.index("_") +
1:file_name.rindex(".")]
AU_column_idx = {}
co_occur_AU = {}
with open(label_file_dir + "/" + file_name, "r") as au_file_obj:
for idx, line in enumerate(au_file_obj):
if idx == 0: # header specify Action Unit
for col_idx, AU in enumerate(line.split(",")[1:]):
AU_column_idx[AU] = col_idx + 1 # read header
continue # read head over , continue
lines = line.split(",")
frame = lines[0]
non_label_lst = [
int(lines[AU_column_idx[AU]])
for AU in config.AU_ROI.keys()
if int(AU) >= 40 and int(lines[AU_column_idx[AU]]) != 9
]
if len(non_label_lst) > 0:
print(" big AU is not 9! in {}".format(label_file_dir +
"/" + file_name))
print("{}".format(non_label_lst))
au_label_dict = {AU: int(lines[AU_column_idx[AU]]) for AU in config.AU_ROI.keys() \
if int(lines[AU_column_idx[AU]]) == 1} # 注意只生成=1的字典，而不要=9的字典，就是只要非unknown的AU
for AU_tuple in combinations(au_label_dict.keys(), 2):
co_occur_AU[tuple(sorted(map(int, AU_tuple)))] = 1
all_AU.update(au_label_dict.keys())
au_mask_dict = {AU: "{0}/{1}/{2}/{3}_AU_{4}.png".format(au_mask_dir,
subject_name, sequence_name,
frame, ",".join(au_couple_dict[AU])) \
for AU in au_label_dict.keys()}
for AU, mask_path in au_mask_dict.items():
combine_AU[mask_path].add(AU)
for AU_tuple, count in co_occur_AU.items():
all_co_occur_AU[AU_tuple] += count
AU_counter = defaultdict(int)
for mask_path, combine_AU in combine_AU.items():
AU_counter[tuple(sorted(combine_AU))] += 1
# for AU, count in sorted(AU_counter.items(), key=lambda e: e[1],reverse=True):
# print("{0} {1}".format(AU, count))
return AU_counter, all_AU, all_co_occur_AU
def generate_AUCouple_ROI_mask_image(database_name, img_path):
adaptive_AU_database(database_name)
global MASK_COLOR
mask_color_lst = []
for color in MASK_COLOR:
mask_color_lst.append(color_bgr(color))
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True,
color_bgr(RED),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr, axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
idx = 0
gen_face_lst = dict()
AU_couple_mask = dict()
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in already_fill_AU:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
AU_couple_mask[AU_couple] = mask
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = random.choice(mask_color_lst)
idx += 1
new_face = cv2.addWeighted(cropped_face, 0.75, color_mask, 0.25, 0)
gen_face_lst[AU_couple] = new_face
return gen_face_lst, AU_couple_mask
def generate_AU_ROI_table():
already_AU_couple = set()
AU_couple_dict = get_zip_ROI_AU()
AU_couple_region = OrderedDict()
for AU, region_numbers in sorted(config.AU_ROI.items(), key=lambda e:e[0]):
AU_couple = AU_couple_dict[AU]
if AU_couple not in already_AU_couple:
already_AU_couple.add(AU_couple)
AU_couple_region[AU_couple] = sorted(region_numbers)
for AU_couple, region_numbers in AU_couple_region.items():
AU_info = []
for AU in AU_couple:
AU_info.append("AU {}".format(AU))
AU_couple = " , ".join(AU_info)
region_numbers = " , ".join(map(str, region_numbers))
print("{0} & {1} \\\\".format(AU_couple, region_numbers))
print("\hline")
def generate_AUCouple_ROI_mask_image(self, database_name, img_path,
roi_activate):
adaptive_AU_database(database_name)
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True, (0, 0, 255),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr,
axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
AUCouple_face_dict = dict()
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in already_fill_AU or AU_couple not in roi_activate:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = (199, 21, 133)
new_face = cv2.add(cropped_face, color_mask)
AUCouple_face_dict[AU_couple] = new_face
return AUCouple_face_dict
def collect_box_coordinate(image_path, mc_manager, database):
au_couple_dict = get_zip_ROI_AU()
AU_group_box_coodinate = defaultdict(list) # AU_group => box_list
key_prefix = database + "@512|"
try:
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
image_path,
image_path,
channel_first=True,
mc_manager=mc_manager,
key_prefix=key_prefix)
except IndexError:
return AU_group_box_coodinate
for AU, box_list in AU_box_dict.items():
AU_couple = au_couple_dict[AU]
if AU_couple in AU_group_box_coodinate:
continue
new_box_list = [list(box) for box in box_list]
new_box_list.sort(key=lambda e: e[1])
AU_group_box_coodinate[AU_couple].extend(new_box_list)
return AU_group_box_coodinate
def generate_face_AUregion_mask(database, iter_crop):
landmark = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
AU_couple_dict = get_zip_ROI_AU()
already_mask = set()
for new_face, orig_img, rect, absolute_path in iter_crop: # absolute_path 是截取crop新脸的路径
try:
absolute_path = absolute_path.replace("//", "/")
subject_name = absolute_path.split(os.sep)[-3]
sequence = absolute_path.split(os.sep)[-2]
filename = os.path.basename(absolute_path)
filename = filename[:filename.rindex(".")]
au_mask_dir = "{0}/{1}/{2}/".format(
config.AU_REGION_MASK_PATH[database], subject_name, sequence)
for f in glob.glob("{0}/{1}_AU_*".format(au_mask_dir, filename)):
os.remove(f)
for AU in config.AU_ROI.keys():
if AU_couple_dict[AU] in already_mask:
continue
mask = crop_face_img_mask(AU,
orig_img,
new_face,
rect,
landmarker=landmark)
if not os.path.exists(au_mask_dir):
os.makedirs(au_mask_dir)
print("make dir {}".format(au_mask_dir))
au_couple = AU_couple_dict[AU]
already_mask.add(au_couple)
au_couple = ",".join(au_couple)
au_mask_path = "{0}/{1}_AU_{2}.png".format(
au_mask_dir, filename, au_couple)
cv2.imwrite(au_mask_path, mask)
print("write : {}".format(au_mask_path))
yield mask, au_mask_path
already_mask.clear()
except IndexError:
continue
def stats_AU_group_area(image_path, mc_manager, database):
au_couple_dict = get_zip_ROI_AU()
AU_group_box_area = dict()
key_prefix = database + "|"
try:
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
image_path,
image_path,
channel_first=True,
mc_manager=mc_manager,
key_prefix=key_prefix)
except IndexError:
return AU_group_box_area
for AU, box_list in AU_box_dict.items():
AU_couple = au_couple_dict[AU]
tot_area = 0.
for box in box_list:
y_min, x_min, y_max, x_max = box
area = (x_max - x_min) * (y_max - y_min)
tot_area += area
tot_area /= len(box_list)
AU_group_box_area[AU_couple] = tot_area
return AU_group_box_area
def async_crop_face(database, src_img_dir, mp_num, force_write=True):
pool = mp.Pool()
data_path_lst = set()
not_contain_count = 0
contain_count = 0
AU_couple_dict = get_zip_ROI_AU()
for root, dirs, files in os.walk(src_img_dir):
for file in files:
absolute_path = root + os.sep + file
subject_name = absolute_path.split(os.sep)[-3]
sequence = absolute_path.split(os.sep)[-2]
dst_dir = config.CROP_DATA_PATH[
database] + os.sep + subject_name + os.sep + sequence + os.sep
cropped_path = dst_dir + os.sep + os.path.basename(absolute_path)
data_path_lst.add(absolute_path)
split_list = lambda A, n=3: [A[i:i + n] for i in range(0, len(A), n)]
print(len(data_path_lst) // mp_num)
sub_list = split_list(list(data_path_lst), len(data_path_lst) // mp_num)
for sub in sub_list:
pool.apply_async(sub_process, args=(database, sub, force_write))
pool.close()
pool.join()
data_base = options.database
data_path = options.data_path
if data_path is None:
opt.error("error, data_path not given!")
min_support = options.min_support
if min_support is None:
opt.error("error, min_support not given!")
if data_base == "BP4D":
data = read_BP4D_AU_file(data_path)
elif data_base == "DISFA":
data = read_BP4D_AU_file(data_path)
import pyfpgrowth
from collections import defaultdict
import config
from dataset_toolkit.compress_utils import get_zip_ROI_AU, get_AU_couple_child
au_couple_dict = get_zip_ROI_AU()
roi_set = set(list(au_couple_dict.values()))
AU_belong_roi_id = {"0": 999}
for idx, couple in enumerate(list(set(au_couple_dict.values()))):
for AU in couple:
AU_belong_roi_id[AU] = idx
DISFA_roi_AU = defaultdict(list)
if data_base == "DISFA":
all_print_tuple = set()
for AU in config.DISFA_use_AU:
DISFA_roi_AU[AU_belong_roi_id[AU]].append(AU)
for entry in itertools.product(*DISFA_roi_AU.values()):
print_tuple = list(map(str, sorted(map(int, list(entry)))))
while len(print_tuple) < config.BOX_NUM[data_base]:
AU_1 = random.choice(["1", "2", "5"])
def main():
parser = argparse.ArgumentParser(
description='generate Graph desc file script')
parser.add_argument('--mean',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument("--image",
default='C:/Users/machen/Downloads/tmp/face.jpg')
parser.add_argument(
"--model", default="C:/Users/machen/Downloads/tmp/BP4D_3_fold_1.npz")
parser.add_argument("--pretrained_model_name",
'-premodel',
default='resnet101')
parser.add_argument('--database', default='BP4D', help='Output directory')
parser.add_argument('--device',
default=0,
type=int,
help='GPU device number')
args = parser.parse_args()
adaptive_AU_database(args.database)
if args.pretrained_model_name == "resnet101":
faster_rcnn = FasterRCNNResnet101(
n_fg_class=len(config.AU_SQUEEZE),
pretrained_model="resnet101",
mean_file=args.mean,
use_lstm=False,
extract_len=1000
) # 可改为/home/machen/face_expr/result/snapshot_model.npz
elif args.pretrained_model_name == "vgg":
faster_rcnn = FasterRCNNVGG16(n_fg_class=len(config.AU_SQUEEZE),
pretrained_model="imagenet",
mean_file=args.mean,
use_lstm=False,
extract_len=1000)
if os.path.exists(args.model):
print("loading pretrained snapshot:{}".format(args.model))
chainer.serializers.load_npz(args.model, faster_rcnn)
if args.device >= 0:
faster_rcnn.to_gpu(args.device)
chainer.cuda.get_device_from_id(int(args.device)).use()
heatmap_gen = HeatMapGenerator(np.load(args.model), use_relu=True)
if args.device >= 0:
heatmap_gen.to_gpu(args.device)
cropped_face, AU_box_dict = FaceMaskCropper.get_cropface_and_box(
args.image, args.image, channel_first=True)
au_couple_dict = get_zip_ROI_AU()
au_couple_child = get_AU_couple_child(
au_couple_dict) # AU couple tuple => child fetch list
au_couple_box = dict() # value is box (4 tuple coordinate) list
for AU, AU_couple in au_couple_dict.items():
au_couple_box[AU_couple] = AU_box_dict[AU]
box_lst = []
roi_no_AU_couple_dict = dict()
roi_no = 0
for AU_couple, couple_box_lst in au_couple_box.items():
box_lst.extend(couple_box_lst)
for _ in couple_box_lst:
roi_no_AU_couple_dict[roi_no] = AU_couple
roi_no += 1
box_lst = np.asarray(box_lst)
cropped_face = cropped_face.astype(np.float32)
orig_face = cropped_face
cropped_face = faster_rcnn.prepare(
cropped_face) # substract mean pixel value
box_lst = box_lst.astype(np.float32)
orig_box_lst = box_lst
batch = [
(cropped_face, box_lst),
]
cropped_face, box_lst = concat_examples(
batch, args.device) # N,3, H, W, ; N, F, 4
if box_lst.shape[1] != config.BOX_NUM[args.database]:
print("error box num {0} != {1}".format(box_lst.shape[1],
config.BOX_NUM[args.database]))
return
with chainer.no_backprop_mode(), chainer.using_config("train", False):
cropped_face = chainer.Variable(cropped_face)
box_lst = chainer.Variable(box_lst)
roi_preds, _ = faster_rcnn.predict(cropped_face, box_lst) # R, 22
roi_feature_maps = faster_rcnn.extract(orig_face, orig_box_lst,
'res5') # R, 2048 7,7
roi_images = []
box_lst = box_lst[0].data.astype(np.int32)
for box in box_lst:
y_min, x_min, y_max, x_max = box
roi_image = orig_face[:, y_min:y_max + 1,
x_min:x_max + 1] # N, 3, roi_H, roi_W
roi_images.append(roi_image) # list of N, 3, roi_H, roi_W
cmap = plt.get_cmap('jet')
# image_activate_map = np.zeros((cropped_face.shape[2], cropped_face.shape[3]), dtype=np.float32)
for box_id, (roi_image, roi_feature_map) in enumerate(
zip(roi_images, roi_feature_maps)):
y_min, x_min, y_max, x_max = box_lst[box_id]
# 22, roi_h, roi_w, 3
xp = chainer.cuda.get_array_module(roi_feature_map)
roi_feature_map = xp.expand_dims(roi_feature_map, 0)
# class_roi_overlay_img = 22, roi_h, roi_w
class_roi_activate_img = heatmap_gen.generate_activate_roi_map(
roi_feature_map, (y_max - y_min + 1, x_max - x_min + 1))
roi_pred = roi_preds[box_id] # 22
# choice_activate_map = np.zeros((y_max-y_min+1, x_max-x_min+1), dtype=np.float32)
# use_choice = False
if len(np.nonzero(roi_pred)
[0]) > 0: # TODO : 还要做做 class的选择，以及 heatmap采用cv2.add的模式相加
class_idx = random.choice(np.nonzero(roi_pred)[0])
AU = config.AU_SQUEEZE[class_idx]
print(AU)
choice_activate_map = class_roi_activate_img[
class_idx] # roi_h, roi_w
activation_color_map = np.round(
cmap(choice_activate_map)[:, :, :3] * 255).astype(np.uint8)
overlay_img = roi_images[
box_id] / 2 + activation_color_map.transpose(2, 0, 1) / 2
overlay_img = np.transpose(overlay_img,
(1, 2, 0)).astype(np.uint8)
vis_img = cv2.cvtColor(overlay_img, cv2.COLOR_RGB2BGR)
cv2.imshow("new", vis_img)
cv2.waitKey(0)
def build_graph_roi_single_label(faster_rcnn, reader_func, output_dir,
database_name, force_generate, proc_num,
cut: bool, extract_key, train_subject,
test_subject):
'''
currently CRF can only deal with single label situation
so use /home/machen/dataset/BP4D/label_dict.txt to regard combine label as new single label
example(each file contains one video!):
node_id kown_label features
1_12 +1 np_file:/path/to/npy features:1,3,4,5,5,...
node_id specific: ${frame}_${roi}, eg: 1_12
or
444 +[0,0,0,1,0,1,0] np_file:/path/to/npy features:1,3,4,5,5,...
spatio can have two factor node here, for example spatio_1 means upper face, and spatio_2 means lower face relation
#edge 143 4289 spatio_1
#edge 143 4289 spatio_2
#edge 112 1392 temporal
mode: RNN or CRF
'''
adaptive_AU_database(database_name)
adaptive_AU_relation(database_name)
au_couple_dict = get_zip_ROI_AU(
) # value is AU couple tuple, each tuple denotes an RoI
# max_au_couple_len = max(len(couple) for couple in au_couple_dict.values()) # we use itertools.product instead
label_bin_len = config.BOX_NUM[
database_name] # each box/ROI only have 1 or 0
au_couple_set = set(au_couple_dict.values())
au_couple_list = list(au_couple_set)
au_couple_list.append(("1", "2", "5", "7")) # because it is symmetric area
is_binary_AU = True
for video_info, subject_id in reader_func(
output_dir,
is_binary_AU=is_binary_AU,
is_need_adaptive_AU_relation=False,
force_generate=force_generate,
proc_num=proc_num,
cut=cut,
train_subject=train_subject):
extracted_feature_cache = dict(
) # key = np.ndarray_hash , value = h. speed up
frame_box_cache = dict() # key = frame, value = boxes
frame_labels_cache = dict()
frame_AU_couple_bbox_dict_cache = dict()
# each video file is copying multiple version but differ in label
if database_name == "BP4D":
label_split_list = config.BP4D_LABEL_SPLIT
elif database_name == "DISFA":
label_split_list = config.DISFA_LABEL_SPLIT
for couples_tuple in label_split_list: # couples_tuple = ("1","3","5",.."4") cross AU_couple, config.LABEL_SPLIT come from frequent pattern statistics
assert len(couples_tuple) == config.BOX_NUM[database_name]
couples_tuple = tuple(map(str, sorted(map(int, couples_tuple))))
couples_tuple_set = set(
couples_tuple) # use cartesian product to iterator over
if len(couples_tuple_set) < len(couples_tuple):
continue
# limit too many combination
# count = 0
# for fp in fp_set:
# inter_set = couples_tuple_set & set(fp)
# union_set = couples_tuple_set | set(fp)
# iou = len(inter_set) / len(union_set)
# if iou > 0.6:
# count += 1
# if count < 20:
# continue
node_list = []
temporal_edges = []
spatio_edges = []
h_info_array = []
box_geometry_array = []
for entry_dict in video_info:
frame = entry_dict["frame"]
cropped_face = entry_dict["cropped_face"]
print("processing frame:{}".format(frame))
all_couple_mask_dict = entry_dict[
"all_couple_mask_dict"] # key is AU couple tuple,不管脸上有没有该AU都返回回来
image_labels = entry_dict[
"all_labels"] # each region has a label(binary or AU)
bboxes = []
labels = []
AU_couple_bbox_dict = OrderedDict()
if frame in frame_box_cache:
bboxes = frame_box_cache[frame]
labels = frame_labels_cache[frame]
AU_couple_bbox_dict = frame_AU_couple_bbox_dict_cache[
frame]
else:
for idx, (AU_couple, mask) in enumerate(
all_couple_mask_dict.items()
): # We cannot sort this dict here, because region_label depend on order of this dict.AU may contain single_true AU or AU binary tuple (depends on need_adaptive_AU_relation)
region_label = image_labels[
idx] # str or tuple, so all_labels index must be the same as all_couple_mask_dict
connect_arr = cv2.connectedComponents(mask,
connectivity=8,
ltype=cv2.CV_32S)
component_num = connect_arr[0]
label_matrix = connect_arr[1]
for component_label in range(1, component_num):
row_col = list(
zip(*np.where(
label_matrix == component_label)))
row_col = np.array(row_col)
y_min_index = np.argmin(row_col[:, 0])
y_min = row_col[y_min_index, 0]
x_min_index = np.argmin(row_col[:, 1])
x_min = row_col[x_min_index, 1]
y_max_index = np.argmax(row_col[:, 0])
y_max = row_col[y_max_index, 0]
x_max_index = np.argmax(row_col[:, 1])
x_max = row_col[x_max_index, 1]
# same region may be shared by different AU, we must deal with it
coordinates = (y_min, x_min, y_max, x_max)
if y_min == y_max and x_min == x_max:
continue
if coordinates not in bboxes:
bboxes.append(
coordinates
) # bboxes and labels have the same order
labels.append(
region_label
) # AU may contain single_true AU or AU binary tuple (depends on need_adaptive_AU_relation)
AU_couple_bbox_dict[coordinates] = AU_couple
del label_matrix
if len(bboxes) != config.BOX_NUM[database_name]:
print("boxes num != {0}, real box num= {1}".format(
config.BOX_NUM[database_name], len(bboxes)))
continue
frame_box_cache[frame] = bboxes
frame_AU_couple_bbox_dict_cache[frame] = AU_couple_bbox_dict
frame_labels_cache[frame] = labels
box_idx_AU_dict = dict(
) # box_idx => AU, cannot cache! because couples_tuple each time is different
already_added_AU_set = set()
for box_idx, _ in enumerate(bboxes): # bboxes may from cache
AU_couple = list(AU_couple_bbox_dict.values())[
box_idx] # AU_couple_bbox_dict may from cache
for AU in couples_tuple: # couples_tuple not from cache, thus change after each iteration 每轮迭代完的时候变换
if AU in AU_couple and AU not in already_added_AU_set:
box_idx_AU_dict[box_idx] = (AU, AU_couple)
already_added_AU_set.add(AU)
break
cropped_face.flags.writeable = False
key = hash(cropped_face.data.tobytes())
if key in extracted_feature_cache:
h = extracted_feature_cache[key]
else:
with chainer.no_backprop_mode(), chainer.using_config(
'train', False):
h = faster_rcnn.extract(
cropped_face, bboxes,
layer=extract_key) # shape = R' x 2048
extracted_feature_cache[key] = h
assert h.shape[0] == len(bboxes)
h = chainer.cuda.to_cpu(h)
h = h.reshape(len(bboxes), -1)
# 这个indent级别都是同一张图片内部
# print("box number, all_mask:", len(bboxes),len(all_couple_mask_dict))
assert len(box_idx_AU_dict) == config.BOX_NUM[database_name]
for box_idx, (AU,
AU_couple) in sorted(box_idx_AU_dict.items(),
key=lambda e: int(e[0])):
label = np.zeros(
shape=label_bin_len, dtype=np.int32
) # bin length became box number > AU_couple number
AU_squeeze_idx = config.AU_SQUEEZE.inv[AU]
label[couples_tuple.index(AU)] = labels[box_idx][
AU_squeeze_idx] # labels缓存起来可能出错 # labels[box_idx] = 0,0,1,1,...,0 but we want only look at specific idx
label = tuple(label)
label_arr = np.char.mod("%d", label)
label = "({})".format(",".join(label_arr))
h_flat = h[box_idx]
node_id = "{0}_{1}".format(frame, box_idx)
node_list.append(
"{0} {1} feature_idx:{2} AU_couple:{3} AU:{4}".format(
node_id, label, len(h_info_array), AU_couple, AU))
h_info_array.append(h_flat)
box_geometry_array.append(bboxes[box_idx])
# 同一张画面两两组合，看有没连接线，注意AU=0，就是未出现的AU动作的区域也参与连接
for box_idx_a, box_idx_b in map(
sorted, itertools.combinations(range(len(bboxes)), 2)):
node_id_a = "{0}_{1}".format(frame, box_idx_a)
node_id_b = "{0}_{1}".format(frame, box_idx_b)
AU_couple_a = AU_couple_bbox_dict[bboxes[
box_idx_a]] # AU couple represent region( maybe symmetry in face)
AU_couple_b = AU_couple_bbox_dict[bboxes[box_idx_b]]
if AU_couple_a == AU_couple_b or has_edge(
AU_couple_a, AU_couple_b, database_name):
spatio_edges.append("#edge {0} {1} spatio".format(
node_id_a, node_id_b))
box_id_temporal_dict = defaultdict(
list) # key = roi/bbox id, value = node_id list cross temporal
for node_info in node_list:
node_id = node_info[0:node_info.index(" ")]
box_id = node_id[node_id.index("_") + 1:]
box_id_temporal_dict[box_id].append(node_id)
for node_id_list in box_id_temporal_dict.values():
for idx, node_id in enumerate(node_id_list):
if idx + 1 < len(node_id_list):
node_id_next = node_id_list[idx + 1]
temporal_edges.append("#edge {0} {1} temporal".format(
node_id, node_id_next))
train_AU_out_path = "{0}/train/{1}/{2}.txt".format(
output_dir, "_".join(map(str, couples_tuple)),
video_info[0]["video_id"])
test_AU_out_path = "{0}/test/{1}/{2}.txt".format(
output_dir, "_".join(map(str, couples_tuple)),
video_info[0]["video_id"])
if subject_id in train_subject:
output_path = train_AU_out_path
npz_path = output_dir + os.sep + "train" + os.sep + os.path.basename(
output_path)[:os.path.basename(output_path).
rindex(".")] + ".npz"
elif subject_id in test_subject:
output_path = test_AU_out_path
npz_path = output_dir + os.sep + "test" + os.sep + os.path.basename(
output_path)[:os.path.basename(output_path).
rindex(".")] + ".npz"
os.makedirs(os.path.dirname(output_path), exist_ok=True)
if not os.path.exists(npz_path):
np.savez(npz_path,
appearance_features=h_info_array,
geometry_features=np.array(box_geometry_array,
dtype=np.float32))
with open(output_path, "w") as file_obj:
for line in node_list:
file_obj.write("{}\n".format(line))
for line in spatio_edges:
file_obj.write("{}\n".format(line))
for line in temporal_edges:
file_obj.write("{}\n".format(line))
file_obj.flush()
node_list.clear()
spatio_edges.clear()
temporal_edges.clear()
h_info_array.clear()
def read_DISFA_video_label(output_dir,
is_binary_AU,
is_need_adaptive_AU_relation=False,
force_generate=True,
proc_num=10,
cut=False,
train_subject=None):
mgr = mp.Manager()
queue = mgr.Queue(maxsize=20000)
for orientation in ["Left", "Right"]:
if is_need_adaptive_AU_relation:
adaptive_AU_relation(
) # delete AU relation pair occur in same facial region
au_couple_dict = get_zip_ROI_AU()
au_couple_child_dict = get_AU_couple_child(au_couple_dict)
DISFA_base_dir = config.DATA_PATH["DISFA"]
label_file_dir = DISFA_base_dir + "/ActionUnit_Labels/"
img_folder = DISFA_base_dir + "/Img_{}Camera".format(orientation)
for video_name in os.listdir(label_file_dir): # each file is a video
is_train = True if video_name in train_subject else False
if not force_generate:
prefix = "train" if is_train else "test"
target_file_path = output_dir + os.sep + prefix + os.sep + video_name + "_" + orientation + ".npz"
if os.path.exists(target_file_path):
continue
resultdict = {}
if proc_num > 1:
pool = mp.Pool(processes=proc_num)
procs = 0
one_file_name = os.listdir(label_file_dir + os.sep +
video_name)[0]
with open(
label_file_dir + os.sep + video_name + os.sep +
one_file_name, "r") as file_obj:
for idx, line in enumerate(file_obj):
line = line.strip()
if line:
frame = line.split(",")[0]
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
pool.apply_async(func=delegate_mask_crop,
args=(img_path, True, queue))
procs += 1
for i in range(procs):
try:
entry = queue.get(block=True, timeout=60)
resultdict[entry[0]] = (entry[1], entry[2])
except Exception:
print("queue block time out")
break
pool.close()
pool.join()
del pool
else: # only one process
one_file_name = os.listdir(label_file_dir + os.sep +
video_name)[0]
with open(
label_file_dir + os.sep + video_name + os.sep +
one_file_name, "r") as file_obj:
for idx, line in enumerate(file_obj):
line = line.strip()
if line:
frame = line.split(",")[0]
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, True)
resultdict[img_path] = (cropped_face,
AU_mask_dict)
except IndexError:
pass
frame_label = dict()
video_info = []
video_img_path_set = set()
for file_name in os.listdir(
label_file_dir + os.sep +
video_name): # each file is one AU ( video file )
AU = file_name[file_name.index("au") + 2:file_name.rindex(".")]
with open(
label_file_dir + os.sep + video_name + os.sep +
file_name, "r") as file_obj:
for line in file_obj:
frame = int(line.split(",")[0])
AU_intensity = int(line.split(",")[1])
img_path = img_folder + "/{0}/{1}.jpg".format(
video_name, frame)
video_img_path_set.add((frame, img_path))
if frame not in frame_label:
frame_label[frame] = set()
if AU_intensity >= 1: # NOTE that we apply AU_intensity >= 1
frame_label[int(frame)].add(AU) #存储str类型
for frame, img_path in sorted(video_img_path_set,
key=lambda e: int(e[0])):
if img_path not in resultdict:
continue
AU_set = frame_label[frame] # it is whole image's AU set
if cut and len(AU_set) == 0:
continue
cropped_face, AU_mask_dict = resultdict[img_path]
all_couple_mask_dict = OrderedDict()
for AU in sorted(map(
int, config.AU_ROI.keys())): # ensure same order
all_couple_mask_dict[au_couple_dict[str(
AU)]] = AU_mask_dict[str(AU)]
all_labels = list() # 开始拼接all_labels
for AU_couple in all_couple_mask_dict.keys(
): # 顺序与all_couple_mask_dict一致
child_AU_couple_list = au_couple_child_dict[AU_couple]
AU_couple = set(AU_couple)
for child_AU_couple in child_AU_couple_list:
AU_couple.update(
child_AU_couple) # combine child region's AU
if not is_binary_AU: # in CRF, CRF模式需要将同一个区域的多个AU用逗号分隔，拼接
concat_AU = []
for AU in AU_couple:
if AU in AU_set: # AU_set 存储真实AU(ground truth label):str类型
concat_AU.append(AU)
if len(concat_AU) == 0:
all_labels.append(
"0") # 若该区域压根没有任何AU出现，为了让只支持单label的CRF工作，用0来代替
else:
all_labels.append(",".join(sorted(concat_AU)))
else: # convert to np.array which is AU_bin
AU_bin = np.zeros(len(config.AU_SQUEEZE)).astype(
np.uint8)
for AU in AU_couple:
if AU in AU_set: # judge if this region contain which subset of whole image's AU_set
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], 1)
all_labels.append(tuple(AU_bin))
video_info.append({
"frame": frame,
"cropped_face": cropped_face,
"all_couple_mask_dict": all_couple_mask_dict,
"all_labels": all_labels,
"video_id": video_name + "_" + orientation
})
resultdict.clear()
if video_info:
yield video_info, video_name
else:
print(
"error in file:{} no video found".format(video_name + "_" +
orientation))
def read_BP4D_video_label(output_dir,
is_binary_AU,
is_need_adaptive_AU_relation=False,
force_generate=True,
proc_num=10,
cut=False,
train_subject=None):
'''
:param
output_dir : 用于检查如果目标的文件已经存在，那么就不再生成
is_binary_AU:
True --> return AU_binary 01010100
False --> used for CRF mode: single true AU label CRF/ or AU combination separate by comma
:yield: 每个视频video收集齐了yield回去，视频中每一帧返回3部分：
1. "img_path": /path/to/image
1."all_couple_mask_dict": 是OrderedDict，包含所有区域的mask，不管AU是不是+1，还是-1(不管AU出现没出现)，key是AU_couple，来自于au_couple_dict = get_zip_ROI_AU()
2."labels": 是list，index与all_couple_mask_dict一致，其中每个label
要么是binary形式01010110，
要么是3,4（由于一块位置可以发生多个AU，因此可以用逗号隔开的字符串来返回），根据is_binary_AU返回不同的值
'''
mgr = mp.Manager()
queue = mgr.Queue(maxsize=20000)
if is_need_adaptive_AU_relation:
adaptive_AU_relation(
) # delete AU relation pair occur in same facial region
au_couple_dict = get_zip_ROI_AU()
au_couple_child_dict = get_AU_couple_child(
au_couple_dict) # AU_couple => list of child AU_couple
# if need_translate_combine_AU ==> "mask_path_dict":{(2,3,4): /pathtomask} convert to "mask_path_dict":{110: /pathtomask}
# each is dict : {"img": /path/to/img, "mask_path_dict":{(2,3,4): /pathtomask}, }
BP4D_base_dir_path = config.DATA_PATH["BP4D"]
label_file_dir = BP4D_base_dir_path + "/AUCoding/"
for file_name in os.listdir(label_file_dir): # each file is a video
subject_name = file_name[:file_name.index("_")]
sequence_name = file_name[file_name.index("_") +
1:file_name.rindex(".")]
is_train = True if subject_name in train_subject else False
if not force_generate:
prefix = "train" if is_train else "test"
target_file_path = output_dir + os.sep + prefix + os.sep + subject_name + "_" + sequence_name + ".npz"
if os.path.exists(target_file_path):
continue
resultdict = {}
if proc_num > 1:
one_image_path = os.listdir(config.RGB_PATH["BP4D"] + os.sep +
subject_name + os.sep +
sequence_name)[0]
zfill_len = len(one_image_path[:one_image_path.rindex(".")])
procs = 0
# read image file and crop and get AU mask
pool = mp.Pool(processes=proc_num)
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
for idx, line in enumerate(au_file_obj):
if idx == 0:
continue
lines = line.split(",")
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
pool.apply_async(func=delegate_mask_crop,
args=(img_path, True, queue))
procs += 1
# p = mp.Process(target=delegate_mask_crop, args=(img_path, True, queue))
# procs.append(p)
# p.start()
for i in range(procs):
try:
entry = queue.get(block=True, timeout=360)
resultdict[entry[0]] = (entry[1], entry[2])
except Exception:
print("queue block time out")
break
pool.close()
pool.join()
del pool
else: # only one process
one_image_path = os.listdir(config.RGB_PATH["BP4D"] + os.sep +
subject_name + os.sep +
sequence_name)[0]
zfill_len = len(one_image_path[:one_image_path.rindex(".")])
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
for idx, line in enumerate(au_file_obj):
lines = line.split(",")
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
try:
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=True)
# note that above AU_mask_dict, each AU may have mask that contains multiple separate regions
resultdict[img_path] = (cropped_face, AU_mask_dict)
print("one image :{} done".format(img_path))
except IndexError:
print("img_path:{} cannot obtain 68 landmark".format(
img_path))
pass
# for p in procs:
# p.join()
AU_column_idx = {}
with open(label_file_dir + "/" + file_name,
"r") as au_file_obj: # each file is a video
video_info = []
for idx, line in enumerate(
au_file_obj): # each line represent a frame image
line = line.rstrip()
lines = line.split(",")
if idx == 0: # header define which column is which Action Unit
for col_idx, AU in enumerate(lines[1:]):
AU_column_idx[AU] = col_idx + 1 # read header
continue # read head over , continue
frame = lines[0].zfill(zfill_len)
img_path = config.RGB_PATH[
"BP4D"] + os.sep + subject_name + os.sep + sequence_name + os.sep + frame + ".jpg"
if not os.path.exists(img_path):
print("not exists img_path:{}".format(img_path))
continue
if img_path not in resultdict:
print("img_path:{} landmark not found, continue".format(
img_path))
continue
cropped_face, AU_mask_dict = resultdict[img_path]
all_couple_mask_dict = OrderedDict()
for AU in sorted(map(
int, config.AU_ROI.keys())): # ensure same order
all_couple_mask_dict[au_couple_dict[str(
AU)]] = AU_mask_dict[str(AU)]
au_label_dict = {
AU: int(lines[AU_column_idx[AU]])
for AU in config.AU_ROI.keys()
} # store real AU label
if cut and all(_au_label == 0
for _au_label in au_label_dict.values()):
continue
all_labels = list() # 开始拼接all_labels
for AU_couple in all_couple_mask_dict.keys(
): # 顺序与all_couple_mask_dict一致
child_AU_couple_list = au_couple_child_dict[AU_couple]
AU_couple = set(AU_couple)
for child_AU_couple in child_AU_couple_list:
AU_couple.update(
child_AU_couple
) # label fetch: combine child region's AU
if not is_binary_AU: # in CRF, CRF模式需要将同一个区域的多个AU用逗号分隔，拼接
concat_AU = []
for AU in AU_couple:
if au_label_dict[AU] == 1:
concat_AU.append(AU)
elif au_label_dict[AU] == 9:
concat_AU.append("?{}".format(AU))
if len(concat_AU) == 0:
all_labels.append(
"0") # 若该区域压根没有任何AU出现，为了让只支持单label的CRF工作，用0来代替
else:
all_labels.append(",".join(concat_AU))
else: # convert to np.array which is AU_bin
AU_bin = np.zeros(len(config.AU_SQUEEZE)).astype(
np.uint8)
for AU in AU_couple:
if au_label_dict[AU] == 9:
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], -1)
elif au_label_dict[AU] == 1:
np.put(AU_bin, config.AU_SQUEEZE.inv[AU], 1)
all_labels.append(tuple(AU_bin))
video_info.append({
"frame":
frame,
"cropped_face":
cropped_face,
"all_couple_mask_dict":
all_couple_mask_dict,
"all_labels":
all_labels,
"video_id":
subject_name + "_" + sequence_name
})
resultdict.clear()
if video_info:
yield video_info, subject_name
else:
print("error video_info:{}".format(file_name))
def generate_mask_contain_img(database_name, img_path):
adaptive_AU_database(database_name)
mask_color = {}
for parent_color, child_color in MASK_CONTAIN.items():
mask_color[color_bgr(parent_color)] = color_bgr(child_color)
cropped_face, AU_mask_dict = FaceMaskCropper.get_cropface_and_mask(
img_path, channel_first=False)
AU_couple_dict = get_zip_ROI_AU()
AU_couple_child = get_AU_couple_child(AU_couple_dict)
land = FaceLandMark(config.DLIB_LANDMARK_PRETRAIN)
landmark, _, _ = land.landmark(image=cropped_face)
roi_polygons = land.split_ROI(landmark)
for roi_no, polygon_vertex_arr in roi_polygons.items():
polygon_vertex_arr[0, :] = np.round(polygon_vertex_arr[0, :])
polygon_vertex_arr[1, :] = np.round(polygon_vertex_arr[1, :])
polygon_vertex_arr = sort_clockwise(polygon_vertex_arr.tolist())
cv2.polylines(cropped_face, [polygon_vertex_arr],
True,
color_bgr(RED),
thickness=1)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(cropped_face,
str(roi_no),
tuple(
np.mean(polygon_vertex_arr, axis=0).astype(np.int32)),
font,
0.7, (0, 255, 255),
thickness=1)
already_fill_AU = set()
gen_face_lst = dict()
all_child_set = set()
for child_set in AU_couple_child.values():
for child in child_set:
all_child_set.add(child)
new_face = np.zeros_like(cropped_face)
for AU in config.AU_ROI.keys():
AU_couple = AU_couple_dict[AU]
if AU_couple in all_child_set:
continue
if AU_couple in already_fill_AU:
continue
already_fill_AU.add(AU_couple)
mask = AU_mask_dict[AU]
child_AU_set = AU_couple_child[AU_couple]
color_parent = list(MASK_CONTAIN.keys())[0]
color_child = MASK_CONTAIN[color_parent]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = color_parent
# cv2.addWeighted(color_mask,0.5, color_mask,1-0.5,0,color_mask)
if np.any(new_face):
cropped_face = new_face
cv2.addWeighted(cropped_face, 1, color_mask, 0.3, 0, new_face, -1)
for child_AU in child_AU_set:
if child_AU in already_fill_AU:
continue
already_fill_AU.add(child_AU)
mask = AU_mask_dict[child_AU[0]]
color_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB)
color_mask[mask != 0] = random.choice(color_child)
cv2.addWeighted(new_face, 1, color_mask, 0.5, 0, new_face, -1)
return new_face
