def __getitem__(self, idx):
# load image and label
img_path = os.path.join(self.root, self.imgs[idx])
img = load_img(img_path)
label = self.labels[idx]
label = torch.Tensor([label]).long()
# normalize to [0, 1]
scaler = MinMaxScaler()
img = scaler.fit_transform(img.reshape(-1, img.shape[0])).reshape(
img.shape)
img = torch.Tensor(img)
# data augmentation
target_transform = transforms.Compose([
# TODO: Add resize, change it to actual resnet norm, and add more augmentations
transforms.Resize([224, 224]),
# use actual spec
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
if self.trans is not None:
train_transform = transforms.Compose([
self.trans,
target_transform,
])
else:
train_transform = target_transform
img = train_transform(
img) if idx in self.train_idx else target_transform(img)
return img, label
def load_custom(self, img_dir, csv_file, csv_newfile,
label_map = {'anger':0, 'surprise':5, 'disgust':1, 'fear':2, 'neutral':6, 'happiness':3, 'sadness':4,
'ANGER':0, 'SURPRISE':5, 'DISGUST':1, 'FEAR':2, 'NEUTRAL':6, 'HAPPINESS':3, 'SADNESS':4}):
df = pd.read_csv(csv_file)
bbox = []
net = init_model_dnn()
labels = []
data_path = []
for index, row in df.iterrows():
im = load_img(img_dir + row['image'], False, None)
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
if (box_norm <= [1.0, 1.0, 1.0, 1.0]).all() and label_map[row['emotion']] is not None:
box = box_norm * np.array([w, h, w, h])
bbox.append(box.astype("int"))
labels.append(label_map[row['emotion']])
data_path.append(img_dir + row['image'])
bbox = np.array(bbox)
labels = np.array(labels)
df = pd.DataFrame(data={'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3]})
assert isinstance(csv_newfile, str)
df.to_csv(csv_newfile)
def load_jaffe(self, img_dir, file_format='tiff',
label_map = {'NE' : 'neutral', 'AN' : 'anger', 'DI' : 'disgust', 'FE' : 'fear', 'HA' : 'happy', 'SA' : 'sadness', 'SU' : 'surprise' }):
assert isinstance(img_dir, str) and isinstance(file_format, str)
f_list = glob.glob(img_dir + "/*." + file_format)
labels, data_path = [], []
bbox, bbox_norm = [], []
net = init_model_dnn()
strt_ind = f_list[0].find('jaffe') + 9 # this one is bad
for file in f_list:
im = load_img(file, False, None)
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
bbox.append(box)
bbox_norm.append(box_norm)
# (startX, startY, endX, endY)
data_path.append(file)
labels.append(label_map[file[strt_ind : strt_ind + 2]])
bbox = np.array(bbox)
bbox_norm = np.array(bbox_norm)
self._jaffe_data = {'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3],
'x_norm0' : bbox_norm[:, 0], 'y_norm0' : bbox_norm[:, 1],
'x_norm1' : bbox_norm[:, 2], 'y_norm1' : bbox_norm[:, 3]}
def predict_and_vizualize(csv_filename):
model = Emotion_Net()
model.load_model("models\\ins_resnet_v3_newdata_model.json")
model.load_weights("models\\ins_resnet_v3_newdata_model.h5")
df = pd.read_csv(csv_filename)
smp = df.sample(5, random_state=42)
for index, row in smp.iterrows():
img = load_img(row['file'], False, None)
img = detect_and_classify(img, model)
cv2.imshow('img %d'.format(index), img)
cv2.waitKey(0)
def test_model(model_id, model_folder="models\\"):
model = Emotion_Net()
model.load_model(model_folder + model_id + "model.json")
model.load_weights(model_folder + model_id + "model.h5")
test_imgs = [
'data\\test_img.png', 'data\\test_img2.jpg', 'data\\test_img3.jpg',
'data\\test_img4.jpg'
]
for img_name in test_imgs:
img = load_img(img_name, False, None)
img = detect_and_classify(img, model)
cv2.imshow(img_name, img)
cv2.waitKey(0)
def __getitem__(self, idx):
# load images
img_path = os.path.join(self.root, self.imgs[idx])
img = load_img(img_path)
# normalize to [0, 1]
scaler = MinMaxScaler()
img = scaler.fit_transform(img.reshape(-1, img.shape[0])).reshape(img.shape)
img = torch.Tensor(img)
# load bbox and label
objs = self.data_path[self.data_path['IMG_PATH'] == self.imgs[idx]]
# TODO: change here
labels = objs["CLASS"].map(lambda x: self.class2id[x]).to_numpy()
boxes = objs[["x", "y", "w", "h"]].to_numpy()
# data augmentation
W = img.shape[2]
if idx in self.train_idx and torch.rand(1) < 0.5:
# random horizontal flip with prob 0.5
img = F.hflip(img)
boxes[:, 0] = W - boxes[:, 0] - boxes[:, 2]
# data augmentation
target_transform = transforms.Compose([
# TODO: Add resize, change it to actual resnet norm, and add more augmentations
transforms.Resize([224, 224]),
# use actual spec
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
if self.trans is not None:
train_transform = transforms.Compose([
self.trans,
target_transform,
])
else:
train_transform = target_transform
img = train_transform(img) if idx in self.train_idx else target_transform(img)
# target dict of tensors
target = {
"boxes": torch.Tensor(boxes),
"labels": torch.Tensor(labels).long(),
}
return img, target
def load_facesdb(self, img_dir, file_format='tif',
label_map = {0 : 'neutral', 1: 'happy', 2 : 'sadness', 3 : 'surprise', 4: 'anger', 5 : 'disgust', 6 : 'fear'}):
# label mapping no contempt with contempt | New data
# 0 - neutral -> 0 -> 0 | 6
# 1 - happy -> 4 -> 5 | 3
# 2 - sadness -> 5 -> 6 | 4
# 3 - surprise -> 6 -> 7 | 5
# 4 - anger -> 1 -> 1 | 0
# 5 - disgust -> 2 -> 3 | 1
# 6 - fear -> 3 -> 4 | 2
# 9 - kiss
assert isinstance(img_dir, str) and isinstance(file_format, str)
labels, data_path = [], []
bbox, bbox_norm = [], []
net = init_model_dnn()
for root, dirs, files in os.walk(img_dir):
for file in files:
strt_ind = file.find('_img.' + file_format) - 2
label = int(file[strt_ind : strt_ind + 2])
if label <= 6:
file_path = os.path.join(root, file)
im = load_img(file_path, False, None)
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
bbox.append(box)
bbox_norm.append(box_norm)
data_path.append(file_path)
labels.append(label_map[label])
bbox = np.array(bbox)
bbox_norm = np.array(bbox_norm)
self._facesdb_data = {'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3],
'x_norm0' : bbox_norm[:, 0], 'y_norm0' : bbox_norm[:, 1],
'x_norm1' : bbox_norm[:, 2], 'y_norm1' : bbox_norm[:, 3]}
def __getitem__(self, idx):
# load image and label
img_path = os.path.join(self.root, self.imgs[idx])
img = load_img(img_path)
# normalize to [0, 1]
scaler = MinMaxScaler()
img = scaler.fit_transform(img.reshape(-1, img.shape[0])).reshape(
img.shape)
img = torch.Tensor(img)
# data augmentation
target_transform = transforms.Compose([
transforms.Resize([224, 224]),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
img = target_transform(img)
return img
def preproses(self, img_dir, csv_filename):
self._datafr = pd.read_csv(csv_filename)
bbox = []
drop_rows = []
net = init_model_dnn()
self._datafr = self._datafr.head(3755)
for index, row in self._datafr.iterrows():
f_path = img_dir + row['name'] + '.jpg'
im = load_img(f_path, False, None)
if im is not None:
row['name'] = f_path
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
box -= (3, 3, 0, 0)
box += (0, 0, 3, 3)
# im = cv2.rectangle(im, (box[0], box[1]), (box[2], box[3]), (0, 0, 255), 2)
# im = cv2.putText(im, str(np.max(detections[0, 0, :, 2])), (box[0], box[1] - 5),
# cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)
# cv2.imshow("test", im)
# cv2.waitKey(0)
bbox.append(box)
else:
drop_rows.append(index)
else:
drop_rows.append(index)
self._datafr.drop(drop_rows)
bbox = np.array(bbox)
self._datafr = self._datafr.assign(bbox_x0 = bbox[:, 0])
self._datafr = self._datafr.assign(bbox_y0 = bbox[:, 1])
self._datafr = self._datafr.assign(bbox_x1 = bbox[:, 2])
self._datafr = self._datafr.assign(bbox_y1 = bbox[:, 3])
def load_kanade(self, label_dir, img_dir, file_format='png',
netral_percentage=0.2,
label_map = {0 : 'neutral', 1: 'anger', 3 : 'disgust', 4 : 'fear', 5: 'happy', 6 : 'sadness', 7 : 'surprise', 2 : None}):
""" Loads kanade dataset
in:
* img_dir - path to database (images);
* label_dir - path to image labels;
* file_format - image format;
Emotion labeling: 0 : neutral, 1 : anger, 2 : contempt,
3 : disgust, 4 : fear, 5 : happy, 6 : sadness, 7 : surprise
"""
assert isinstance(label_dir, str) and isinstance(img_dir, str) and isinstance(file_format, str)
labels, data_path = [], []
bbox, bbox_norm = [], []
net = init_model_dnn()
# Firstly we find out if image is labeled and if yes we load it to the database
for root, dirs, files in os.walk(label_dir):
for file in files:
if '.txt' in file:
labeled_pics_path = os.path.join(root, file)
#? images will be in a similare directory,
#? except file format and root directory will be different
pic_file = file.replace('_emotion.txt', '')
pic_root = root.replace(label_dir, img_dir)
pics_path = os.path.join(pic_root, pic_file + "." + file_format)
pics_neutral_path = os.path.join(pic_root, pic_file[:-2] + "01." + file_format)
im = load_img(pics_path, False, None)
# we fiind face on the image and save it to the database
detections = find_faces_dnn(im, net=net)
confidence_max_pos = np.argmax(detections[0, 0, :, 2])
if detections[0, 0, confidence_max_pos, 2] > self.conf_threshold:
(h, w) = im.shape[:2]
box_norm = detections[0, 0, 0, 3:7]
box = box_norm * np.array([w, h, w, h])
box = box.astype("int")
with open(labeled_pics_path, 'r') as lbl:
labl = float(lbl.read()[:-1])
if labl != 2.0:
bbox.append(box)
bbox_norm.append(box_norm)
data_path.append(pics_path)
if label_map[labl] is not None:
labels.append(label_map[labl])
if random.random() >= netral_percentage:
# saves first neutral frame
bbox.append(box)
bbox_norm.append(box_norm)
data_path.append(pics_neutral_path)
labels.append(label_map[0])
bbox = np.array(bbox)
bbox_norm = np.array(bbox_norm)
self._kanade_data = {'file': data_path, 'label': labels,
'x0' : bbox[:, 0], 'y0' : bbox[:, 1],
'x1' : bbox[:, 2], 'y1' : bbox[:, 3],
'x_norm0' : bbox_norm[:, 0], 'y_norm0' : bbox_norm[:, 1],
'x_norm1' : bbox_norm[:, 2], 'y_norm1' : bbox_norm[:, 3]
}