def unbc(mode='all', use_cropped=False):
root = os.path.join(cfg.root, 'UNBC', '{}')
images = []
annotations = []
keypoints = []
keypoints_norm = []
FACS_Folder = os.path.join('Frame_Labels','FACS')
Images_Folder = 'Images'
Landmarks_Folder = 'AAM_landmarks'
subject_id = sorted(os.listdir(root.format(FACS_Folder)))
num_subjects = len(subject_id)
if mode == 'training':
subject_id = subject_id[:int(round(num_subjects/3.0))]
elif mode == 'validation':
subject_id = subject_id[int(round(num_subjects/3.0)):int(round(num_subjects/3.0))*2]
elif mode == 'test':
subject_id = subject_id[int(round(num_subjects/3.0))*2:]
for subject in subject_id:
subject = os.path.join(root, subject)
sequence_id = sorted(os.listdir(subject.format(FACS_Folder)))
for sequence in sequence_id:
sequence = os.path.join(subject, sequence)
imgs = sorted(glob.glob(os.path.join(sequence.format(Images_Folder), '*png')))
facs = sorted(glob.glob(os.path.join(sequence.format(FACS_Folder), '*txt')))
landmark = sorted(glob.glob(os.path.join(sequence.format(Landmarks_Folder), '*_aam.txt')))
for k in range(len(imgs)):
print "Mode: %s. Extracting dataset from UNBC, image: %s, label:"\
%(mode, imgs[k]),
if k:#First frame belongs to neutral AU: k=0 in neutral
if not os.stat(facs[k]).st_size:
print "Empty File."
continue #If FACS_file is empty
############--------------Read FACS File
images.append(imgs[k])
annotations.append([])
with open(facs[k], 'r') as f:
for batch in read_by_batch(f):
if float(batch)==50.0 or float(batch)==0.0: continue
#print "¿50? no counted ",
#continue#IDK what 50 means on facs files
annotations[-1].append(int(float(batch)))
if not annotations[-1]:
images.pop(-1)
annotations.pop(-1)
print "¿AU50? Empty File."
continue
############--------------Read Landmarks File
kp = []
s_landmarks = [line[:-1].split(' ') for line in open(landmark[k]).readlines()]
for marks in s_landmarks:
kp.append([int(float(marks[0])), int(float(marks[1]))])
kp = kp[17:]#UNBC has unneccesary landmarks
indexes = cfg.indexes0
landmarks_relevant = preparing_data.get_landmarks(kp, indexes)
landmarks_norm = preparing_data.get_landmarks_norm(cv.imread(imgs[k]), landmarks_relevant)
keypoints.append(landmarks_relevant)
keypoints_norm.append(landmarks_norm)
else:
images.append(imgs[0])
annotations.append([0])
kp = []
s_landmarks = [line[:-1].split(' ') for line in open(landmark[0]).readlines()]
for marks in s_landmarks:
kp.append([int(float(marks[0])), int(float(marks[1]))])
kp = kp[17:]#UNBC has unneccesary landmarks
indexes = cfg.indexes0
landmarks_relevant = preparing_data.get_landmarks(kp, indexes)
keypoints.append(landmarks_relevant)
print "%s"%(str(annotations[-1]))
#Check if frames are consecutives. Frames consecutives do not supply any info
n_frames = [int(frames.split('.png')[0][-3:]) for frames in images]
new_images = []
new_annotations = []
new_keypoints = []
consecutives_im = []
consecutives_au = []
consecutives_kp = []
print "Re-extracting dataset for non-consecutive frames"
for item in range(0, len(images)-1):
consecutives_im.append(images[item])
consecutives_au.append(annotations[item])
consecutives_kp.append(keypoints[item])
if (n_frames[item+1] - n_frames[item] == 1) and annotations[item] == annotations[item+1] and not item == len(images)-2:
consecutives_im.append(images[item])
consecutives_au.append(annotations[item])
consecutives_kp.append(keypoints[item])
print "%s consecutive with next one. AU: %s"%(images[item], annotations[item])
else:
new_images.append(consecutives_im[len(consecutives_im)/2])
new_annotations.append(consecutives_au[len(consecutives_au)/2])
new_keypoints.append(consecutives_kp[len(consecutives_kp)/2])
print "%s non-consecutive with %s. AU: %s"%(images[item], images[item].split('/')[-1], annotations[item])
print "Extracted median of these. File: %s. AU: %s"\
%(consecutives_im[len(consecutives_im)/2], consecutives_au[len(consecutives_au)/2])
consecutives_im = []
consecutives_au = []
consecutives_kp = []
print "Transforming relative positions of keypoints to normalized position..."
for item in range(len(new_images)):
landmarks_norm = preparing_data.get_landmarks_norm(cv.imread(new_images[item]), new_keypoints[item])
keypoints_norm.append(landmarks_norm)
if mode == 'cropping':
for file_img in range(len(images)):
new_file = file_img.replace('Images', 'Images_cropped')
folder_file = "/".join(new_file.split('/')[:-1])
os.system('mkdir -p ' + folder_file)
im = cv.imread(images[file_img])
im_crop = croppingImages(im)
if im_crop==None: continue
if os.path.isfile(new_file):
print "File %s already exist"%(new_file)
continue
print "Saving crop image in: %s"%(new_file)
cv.imwrite(new_file, im_crop)
continue
if use_cropped:
images_cp = new_images
for img in range(len(images_cp)):
new_file = new_images[img].replace('Images', 'Images_cropped')
if not os.path.isfile(new_file):
"%s not found"%(new_file)
continue
new_images.append(new_file)
new_annotations.append(new_annotations[img])
img_crop = cv.imread(new_file)
img_org = cv.imread(new_images[img])
new_kp = preparing_data.get_rect_from_cropped(img_org, img_crop, new_keypoints[img])
landmarks_norm = preparing_data.get_landmarks_norm(img_crop, new_kp)
new_keypoints.append(new_kp)
keypoints_norm.append(landmarks_norm)
print "Mode: %s. Extracting dataset from UNBC, image: %s, annotation: %s"\
%(mode, new_images[-1], new_annotations[-1])
if mode == 'cropping': return None
print "Mode: %s. Data collected from UNBC: %d (%d labels), numSubjects: %d"\
%(mode, len(new_images), len(set(get_au_plane(new_annotations))), len(subject_id))
return new_images, new_annotations, new_keypoints, keypoints_norm
def bp4d(mode='all', use_cropped=False):
root = os.path.join(cfg.root, 'BP4D', '{}')
images = []
annotations = []
keypoints = []
keypoints_norm = []
FACS_Folder = os.path.join('AUCoding','AU_OCC')
Images_Folder = 'Sequences'
Landmarks_Folder = '2DFeatures'
Male_subjects = glob.glob(os.path.join(root.format(Images_Folder), 'M*'))
Female_subjects = glob.glob(os.path.join(root.format(Images_Folder), 'F*'))
if mode == 'training':
Male_subjects = Male_subjects[:int(round(len(Male_subjects)/3.0))]
Female_subjects = Female_subjects[:int(round(len(Female_subjects)/3.0))]
elif mode == 'validation':
Male_subjects = Male_subjects[int(round(len(Male_subjects)/3.0)):int(round(len(Male_subjects)/3.0))*2]
Female_subjects = Female_subjects[int(round(len(Female_subjects)/3.0)):int(round(len(Female_subjects)/3.0))*2]
elif mode == 'test':
Male_subjects = Male_subjects[int(round(len(Male_subjects)/3.0))*2:]
Female_subjects = Female_subjects[int(round(len(Female_subjects)/3.0))*2:]
subjects_id = Male_subjects + Female_subjects
facs_files = sorted(glob.glob(os.path.join(root.format(FACS_Folder), '*.csv')))
for csv_file in facs_files:
subject = csv_file.split('/')[-1].split('_')[0]
if not any(sub.split('/')[-1] == subject for sub in subjects_id): continue
emotion = csv_file.split('_')[-1].split('.')[0]
img_folder = os.path.join(root.format(Images_Folder), subject, emotion)
kp_file = os.path.join(root.format(Landmarks_Folder), subject +'_'+ emotion +'.mat')
f_landmark = h5py.File(kp_file, 'r')
file_ = csv.reader(open(csv_file, "rb"))
first_row = True
for row in file_:
if first_row: first_row=False; continue
if os.path.isfile(os.path.join(img_folder, str(row[0]).zfill(4) +'.jpg')):
img_file = os.path.join(img_folder, str(row[0]).zfill(4) +'.jpg')
elif os.path.isfile(os.path.join(img_folder, str(row[0]).zfill(3) +'.jpg')):
img_file = os.path.join(img_folder, str(row[0]).zfill(3) +'.jpg')
else:
img_file = os.path.join(img_folder, str(row[0]).zfill(2) +'.jpg')
row = [int(item) for item in row]
au = np.array(np.where(np.array(row[1:]) == 1))[0].tolist()
if not au: continue
au = [au_+1 for au_ in au]#Since there was removed first column
kp_number = int(img_file.split('/')[-1].split('.')[0])
ff_landmark = np.array(f_landmark[f_landmark['fit']['pred'][kp_number-1][0]])
if not ff_landmark.all(): continue
if 'M005/T7/1359.jpg' in img_file or 'F007/T3/189' in img_file: continue#These images are incomplete
images.append(img_file)
annotations.append(au)
kp = np.hstack((ff_landmark[0].reshape(49,1), ff_landmark[1].reshape(49,1))).tolist()
indexes = cfg.indexes0
landmarks_relevant = preparing_data.get_landmarks(kp, indexes)
keypoints.append(landmarks_relevant)
print "Mode: %s. Extracting dataset from BP4D, image: %s, label: %s. Kpoint file in .mat: %s, img: %d"\
%(mode, img_file, au, kp_file, kp_number)
#Check if frames are consecutives. Frames consecutives do not supply any info
n_frames = []
for frames in images:
n_frames.append(int(frames.split('.jpg')[0].split('/')[-1]))
new_images = []
new_annotations = []
new_keypoints = []
consecutives_im = []
consecutives_au = []
consecutives_kp = []
print "Re-extracting dataset for non-consecutive frames"
for item in range(len(images)-1):
consecutives_im.append(images[item])
consecutives_au.append(annotations[item])
consecutives_kp.append(keypoints[item])
if (n_frames[item+1] - n_frames[item] == 1) and annotations[item] == annotations[item+1] and not item == len(images)-2:
consecutives_im.append(images[item])
consecutives_au.append(annotations[item])
consecutives_kp.append(keypoints[item])
print "%s consecutive with next one. AU: %s"%(images[item], annotations[item])
else:
new_images.append(consecutives_im[len(consecutives_im)/2])
new_annotations.append(consecutives_au[len(consecutives_au)/2])
new_keypoints.append(consecutives_kp[len(consecutives_kp)/2])
print "%s non-consecutive with %s. AU: %s"%(images[item], images[item].split('/')[-1], annotations[item])
print "Extracted median of these. File: %s. AU: %s"\
%(consecutives_im[len(consecutives_im)/2], consecutives_au[len(consecutives_au)/2])
consecutives_im = []
consecutives_au = []
consecutives_kp = []
print "Transforming relative positions of keypoints to normalized position..."
for item in range(len(new_images)):
landmarks_norm = preparing_data.get_landmarks_norm(cv.imread(new_images[item]), new_keypoints[item])
keypoints_norm.append(landmarks_norm)
if mode == 'cropping':
for img_file in range(len(new_images)):
new_dir = new_images[img_file].replace('Sequences', 'Sequences_Cropped')
img_folder = "/".join(new_dir.split('/')[:-1])
if os.path.isfile(new_dir):
print "%s already exist."%(new_dir)
continue
os.system('mkdir -p '+img_folder)
im = cv.imread(new_images[img_file])
if im == None: continue
im_crop = croppingImages(im)
if im_crop==None: continue
print "Saving crop image in: %s."%(new_dir),
cv.imwrite(new_dir, im_crop)
print "Done."
if mode == 'cropping': return None
if use_cropped:
img_cp = new_images
for img_file in range(len(img_cp)):
name = new_images[img_file].replace('Sequences', 'Sequences_Cropped')
if not os.path.isfile(name):
print("%s not found")%(name)
#time.sleep(5)
continue
#raise ValueError('You want to use images cropped, but have not perform cropping. Please use mode: "cropping"')
new_images.append(name)
new_annotations.append(new_annotations[img_file])
img_crop = cv.imread(new_images[img_file])
img_org = cv.imread(name)
new_kp = preparing_data.get_rect_from_cropped(img_org, img_crop, new_keypoints[img_file])
landmarks_norm = preparing_data.get_landmarks_norm(img_crop, new_kp)
new_keypoints.append(new_kp)
keypoints_norm.append(landmarks_norm)
print "Mode: %s. Extracting dataset from BP4D, image: %s, annotation: %s"\
%(mode, new_images[-1], new_annotations[-1])
print "Mode: %s. Data collected from BP4D: %d (%d labels). numSubjects: %d"%(mode, len(new_images), len(set(get_au_plane(new_annotations))), len(subjects_id))
return new_images, new_annotations, new_keypoints, keypoints_norm
def ck(mode = 'all', use_cropped = False):
images = []
annotations = []
keypoints = []
keypoints_norm = []
root = os.path.join(cfg.root, 'CK+', '{}')
root_facs = os.path.join('FACS_labels', 'FACS')
root_images = 'Images'
root_landmarks = 'Landmarks'
list_facs = sorted(os.listdir(root.format(root_facs)))
number_subjects = len(list_facs)
if mode == 'training':
list_facs = list_facs[:int(round(number_subjects/3.0))]
elif mode == 'validation':
list_facs = list_facs[int(round(number_subjects/3.0)):2*int(round(number_subjects/3.0))]
elif mode == 'test':
list_facs = list_facs[int(round(number_subjects/3.0))*2:]
for subject in list_facs:
subj = os.path.join(root, subject)
sessions = os.listdir(subj.format(root_facs))
for session in sessions:
sess = os.path.join(subj, session)
facs_txt = glob.glob(os.path.join(sess.format(root_facs), '*.txt'))
if not facs_txt: continue
lm_txt = sorted(glob.glob(os.path.join(sess.format(root_landmarks), '*.txt')))#last one belongs to action unit
im_file = sorted(glob.glob(os.path.join(sess.format(root_images), '*.png')))
file_img = facs_txt[0].split('/')[-1].split('_facs.txt')[0]+'.png'
file_img = os.path.join(sess.format(root_images), file_img)
if not os.path.isfile(file_img):
print(file_img + ' not found')
#time.sleep(2)
continue
if not im_file[-1] == file_img:
raise ValueError('Label do not match with image file. Label associated with: '+file_img+ 'img: '+im_file)
images.append(file_img)
annotations.append([])
with open(facs_txt[0], 'r') as f:
for batch in read_by_batch(f):
if int(float(batch))==64: continue
annotations[-1].append(int(float(batch)))
landmarks = [line[:-1].split(' ')[1:] for line in open(lm_txt[-1]).readlines()]
kp = []
for line in landmarks:
kp.append([int(float(number)) for number in line])
kp = kp[17:]#CK+ has unneccesary landmarks
indexes = cfg.indexes1
landmarks_relevant = preparing_data.get_landmarks(kp, indexes)
landmarks_norm = preparing_data.get_landmarks_norm(cv.imread(file_img), landmarks_relevant)
keypoints.append(landmarks_relevant)
keypoints_norm.append(landmarks_norm)
print "Mode: %s. Extracting dataset from CK+, image: %s, annotation: %s"\
%(mode, file_img, annotations[-1])
#Annotations for neutral images
file_img_neutral = im_file[0]
annotations.append([0])
landmarks = [line[:-1].split(' ')[1:] for line in open(lm_txt[0]).readlines()]
kp = []
for line in landmarks:
kp.append([int(float(number)) for number in line])
kp = kp[17:]#CK+ has unneccesary landmarks
indexes = cfg.indexes1
landmarks_relevant = preparing_data.get_landmarks(kp, indexes)
landmarks_norm = preparing_data.get_landmarks_norm(cv.imread(file_img_neutral), landmarks_relevant)
keypoints.append(landmarks_relevant)
keypoints_norm.append(landmarks_norm)
images.append(file_img_neutral)
print "Mode: %s. Extracting dataset from CK+, image: %s, annotation: %s"\
%(mode, file_img_neutral, annotations[-1])
if mode == 'cropping':
for file_img in range(len(images)):
new_file = file_img.replace('Images', 'Images_Cropped')
folder_file = "/".join(new_file.split('/')[:-1])
if os.path.isfile(new_file):
print "%s already exist"
continue
im = cv.imread(file_img)
im_crop = croppingImages(im)
if im_crop==None: continue
os.system('mkdir -p '+folder_file)
cv.imwrite(new_file, im_crop)
if mode == 'cropping': return None
if use_cropped:
images_cp = images
for file_img in range(len(images_cp)):
new_file = images[file_img].replace('Images', 'Images_Cropped')
folder_file = "/".join(new_file.split('/')[:-1])
if not os.path.isfile(images[file_img].replace('Images', 'Images_Cropped')):
continue
images.append(new_file)
annotations.append(annotations[file_img])
img_crop = cv.imread(new_file)
img_org = cv.imread(images[file_img])
new_kp = preparing_data.get_rect_from_cropped(img_org, img_crop, keypoints[file_img])
landmarks_norm = preparing_data.get_landmarks_norm(img_crop, new_kp)
keypoints.append(new_kp)
keypoints_norm.append(landmarks_norm)
print "Mode: %s. Extracting dataset from CK+, image: %s, annotation: %s"\
%(mode, images[-1], annotations[-1])
print "Mode: %s. Data collected from CK+: %d (%d labels), numSubjects: %d"%(mode, len(images), \
len(set(get_au_plane(annotations))), len(list_facs))
return images, annotations, keypoints, keypoints_norm