def load_custom(self, img_dir, csv_file, csv_newfile,
label_map = {'anger':0, 'surprise':5, 'disgust':1, 'fear':2, 'neutral':6, 'happiness':3, 'sadness':4,
'ANGER':0, 'SURPRISE':5, 'DISGUST':1, 'FEAR':2, 'NEUTRAL':6, 'HAPPINESS':3, 'SADNESS':4}):
df = pd.read_csv(csv_file)
bbox = []
net = init_model_dnn()
labels = []
data_path = []
for index, row in df.iterrows():
im = load_img(img_dir + row['image'], False, None)
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
if (box_norm <= [1.0, 1.0, 1.0, 1.0]).all() and label_map[row['emotion']] is not None:
box = box_norm * np.array([w, h, w, h])
bbox.append(box.astype("int"))
labels.append(label_map[row['emotion']])
data_path.append(img_dir + row['image'])
bbox = np.array(bbox)
labels = np.array(labels)
df = pd.DataFrame(data={'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3]})
assert isinstance(csv_newfile, str)
df.to_csv(csv_newfile)
def load_jaffe(self, img_dir, file_format='tiff',
label_map = {'NE' : 'neutral', 'AN' : 'anger', 'DI' : 'disgust', 'FE' : 'fear', 'HA' : 'happy', 'SA' : 'sadness', 'SU' : 'surprise' }):
assert isinstance(img_dir, str) and isinstance(file_format, str)
f_list = glob.glob(img_dir + "/*." + file_format)
labels, data_path = [], []
bbox, bbox_norm = [], []
net = init_model_dnn()
strt_ind = f_list[0].find('jaffe') + 9 # this one is bad
for file in f_list:
im = load_img(file, False, None)
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
bbox.append(box)
bbox_norm.append(box_norm)
# (startX, startY, endX, endY)
data_path.append(file)
labels.append(label_map[file[strt_ind : strt_ind + 2]])
bbox = np.array(bbox)
bbox_norm = np.array(bbox_norm)
self._jaffe_data = {'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3],
'x_norm0' : bbox_norm[:, 0], 'y_norm0' : bbox_norm[:, 1],
'x_norm1' : bbox_norm[:, 2], 'y_norm1' : bbox_norm[:, 3]}
def detect_and_classify(
img,
model,
conf_threshold=0.97,
new_size=None,
channels=1,
lbl_map={
0: 'Angry',
1: 'Disgust',
2: 'Fear',
3: 'Happy',
4: 'Sad',
5: 'Surprise',
6: 'Neutral'
}):
"""
0 ----- 'anger',
1 ----- 'disgust'
2 ----- 'fear'
3 ----- 'happiness'
4 ----- 'sadness'
5 ----- 'surprise'
6 ----- 'neutral'
"""
detections = find_faces_dnn(img)
(h, w) = img.shape[:2]
faces = []
bboxes = []
for i in range(0, detections.shape[2]):
confidence = detections[0, 0, i, 2]
if confidence >= conf_threshold:
box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
if (box <= [w, h, w, h]).all() and (box >= [0., 0., 0., 0.]).all():
(startX, startY, endX, endY) = box.astype("int")
face = img[startY:endY, startX:endX]
face = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)
if new_size is not None:
assert len(new_size) == 2
face = cv2.resize(face,
new_size,
interpolation=cv2.INTER_AREA)
faces.append(
np.resize(face, (face.shape[0], face.shape[1], channels)))
bboxes.append([(startX, startY), (endX, endY)])
faces = np.array(faces, dtype='float32')
if faces.shape[0] == 0:
return img
faces = faces / 123.0 - 1
expressions = model.predict(faces)
for i in range(len(bboxes)):
img = cv2.rectangle(img, bboxes[i][0], bboxes[i][1], (0, 0, 255), 2)
img = cv2.putText(img, lbl_map[np.argmax(expressions[i])],
(bboxes[i][0][0], bboxes[i][0][1] - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)
return img
def load_facesdb(self, img_dir, file_format='tif',
label_map = {0 : 'neutral', 1: 'happy', 2 : 'sadness', 3 : 'surprise', 4: 'anger', 5 : 'disgust', 6 : 'fear'}):
# label mapping no contempt with contempt | New data
# 0 - neutral -> 0 -> 0 | 6
# 1 - happy -> 4 -> 5 | 3
# 2 - sadness -> 5 -> 6 | 4
# 3 - surprise -> 6 -> 7 | 5
# 4 - anger -> 1 -> 1 | 0
# 5 - disgust -> 2 -> 3 | 1
# 6 - fear -> 3 -> 4 | 2
# 9 - kiss
assert isinstance(img_dir, str) and isinstance(file_format, str)
labels, data_path = [], []
bbox, bbox_norm = [], []
net = init_model_dnn()
for root, dirs, files in os.walk(img_dir):
for file in files:
strt_ind = file.find('_img.' + file_format) - 2
label = int(file[strt_ind : strt_ind + 2])
if label <= 6:
file_path = os.path.join(root, file)
im = load_img(file_path, False, None)
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
bbox.append(box)
bbox_norm.append(box_norm)
data_path.append(file_path)
labels.append(label_map[label])
bbox = np.array(bbox)
bbox_norm = np.array(bbox_norm)
self._facesdb_data = {'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3],
'x_norm0' : bbox_norm[:, 0], 'y_norm0' : bbox_norm[:, 1],
'x_norm1' : bbox_norm[:, 2], 'y_norm1' : bbox_norm[:, 3]}
def preproses(self, img_dir, csv_filename):
self._datafr = pd.read_csv(csv_filename)
bbox = []
drop_rows = []
net = init_model_dnn()
self._datafr = self._datafr.head(3755)
for index, row in self._datafr.iterrows():
f_path = img_dir + row['name'] + '.jpg'
im = load_img(f_path, False, None)
if im is not None:
row['name'] = f_path
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
box -= (3, 3, 0, 0)
box += (0, 0, 3, 3)
# im = cv2.rectangle(im, (box[0], box[1]), (box[2], box[3]), (0, 0, 255), 2)
# im = cv2.putText(im, str(np.max(detections[0, 0, :, 2])), (box[0], box[1] - 5),
# cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)
# cv2.imshow("test", im)
# cv2.waitKey(0)
bbox.append(box)
else:
drop_rows.append(index)
else:
drop_rows.append(index)
self._datafr.drop(drop_rows)
bbox = np.array(bbox)
self._datafr = self._datafr.assign(bbox_x0 = bbox[:, 0])
self._datafr = self._datafr.assign(bbox_y0 = bbox[:, 1])
self._datafr = self._datafr.assign(bbox_x1 = bbox[:, 2])
self._datafr = self._datafr.assign(bbox_y1 = bbox[:, 3])
def load_and_pross_fer2013(self, csv_file, csv_newfile, im_size = (48, 48)):
df = pd.read_csv(csv_file)
bbox = []
net = init_model_dnn()
for index, row in df.iterrows():
im = np.array([int(x) for x in row['pixels'].split(' ')]).astype('uint8')
im = np.reshape(im, im_size)
im = cv2.cvtColor(im, cv2.COLOR_GRAY2RGB)
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im_size
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
bbox.append(box.astype("int"))
bbox = np.array(bbox)
df = df.assign(x0 = bbox[:, 0])
df = df.assign(y0 = bbox[:, 1])
df = df.assign(x1 = bbox[:, 2])
df = df.assign(y1 = bbox[:, 3])
assert isinstance(csv_newfile, str)
df.to_csv(csv_newfile)
def load_kanade(self, label_dir, img_dir, file_format='png',
netral_percentage=0.2,
label_map = {0 : 'neutral', 1: 'anger', 3 : 'disgust', 4 : 'fear', 5: 'happy', 6 : 'sadness', 7 : 'surprise', 2 : None}):
""" Loads kanade dataset
in:
* img_dir - path to database (images);
* label_dir - path to image labels;
* file_format - image format;
Emotion labeling: 0 : neutral, 1 : anger, 2 : contempt,
3 : disgust, 4 : fear, 5 : happy, 6 : sadness, 7 : surprise
"""
assert isinstance(label_dir, str) and isinstance(img_dir, str) and isinstance(file_format, str)
labels, data_path = [], []
bbox, bbox_norm = [], []
net = init_model_dnn()
# Firstly we find out if image is labeled and if yes we load it to the database
for root, dirs, files in os.walk(label_dir):
for file in files:
if '.txt' in file:
labeled_pics_path = os.path.join(root, file)
#? images will be in a similare directory,
#? except file format and root directory will be different
pic_file = file.replace('_emotion.txt', '')
pic_root = root.replace(label_dir, img_dir)
pics_path = os.path.join(pic_root, pic_file + "." + file_format)
pics_neutral_path = os.path.join(pic_root, pic_file[:-2] + "01." + file_format)
im = load_img(pics_path, False, None)
# we fiind face on the image and save it to the database
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
with open(labeled_pics_path, 'r') as lbl:
labl = float(lbl.read()[:-1])
if labl != 2.0:
bbox.append(box)
bbox_norm.append(box_norm)
data_path.append(pics_path)
if label_map[labl] is not None:
labels.append(label_map[labl])
if random.random() >= netral_percentage:
# saves first neutral frame
bbox.append(box)
bbox_norm.append(box_norm)
data_path.append(pics_neutral_path)
labels.append(label_map[0])
bbox = np.array(bbox)
bbox_norm = np.array(bbox_norm)
self._kanade_data = {'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3],
'x_norm0' : bbox_norm[:, 0], 'y_norm0' : bbox_norm[:, 1],
'x_norm1' : bbox_norm[:, 2], 'y_norm1' : bbox_norm[:, 3]
}