def init_algorithm(
self,
save_training_file='datasets/faces_training.pkl',
load_preprocessed_data='datasets/faces_preprocessed.pkl',
load_mlp='params/mlp.xml',
face_casc='params/haarcascade_frontalface_default.xml',
left_eye_casc='params/haarcascade_lefteye_2splits.xml',
right_eye_casc='params/haarcascade_righteye_2splits.xml'):
self.data_file = save_training_file
self.faces = FaceDetector(face_casc, left_eye_casc, right_eye_casc)
self.head = None
# load preprocessed dataset to access labels and PCA params
if path.isfile(load_preprocessed_data):
(_, y_train), (_, y_test), V, m = homebrew.load_from_file(
load_preprocessed_data)
self.pca_V = V
self.pca_m = m
self.all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array([self.pca_V.shape[1],
len(self.all_labels)])
self.MLP = MultiLayerPerceptron(layer_sizes, self.all_labels)
self.MLP.load(load_mlp)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", load_mlp
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find data file ", load_preprocessed_data
self.testing.Disable()
def init_algorithm(
self,
save_training_file='datasets/faces_training.pkl',
load_preprocessed_data='datasets/faces_preprocessed.pkl',
load_mlp='params/mlp.xml',
face_casc='params/haarcascade_frontalface_default.xml',
left_eye_casc='params/haarcascade_lefteye_2splits.xml',
right_eye_casc='params/haarcascade_righteye_2splits.xml'):
"""Initializes face detector and facial expression classifier
This method initializes both the face detector and the facial
expression classifier.
:param save_training_file: filename for storing the assembled
training set
:param load_preprocessed_data: filename for loading a previously
preprocessed dataset (for
classification in Testing Mode)
:param load_mlp: filename for loading a pre-trained
MLP classifier (use the script
train_test_mlp.py)
:param face_casc: path to a face cascade
:param left_eye_casc: path to a left-eye cascade
:param right_eye_casc: path to a right-eye cascade
"""
self.data_file = save_training_file
self.faces = FaceDetector(face_casc, left_eye_casc, right_eye_casc)
self.head = None
# load preprocessed dataset to access labels and PCA params
if path.isfile(load_preprocessed_data):
(_, y_train), (
_,
y_test), V, m = homebrew.load_from_file(load_preprocessed_data)
self.pca_V = V
self.pca_m = m
self.all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array(
[self.pca_V.shape[1],
len(self.all_labels)])
self.MLP = MultiLayerPerceptron(layer_sizes, self.all_labels)
self.MLP.load(load_mlp)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", load_mlp
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find data file ", load_preprocessed_data
self.testing.Disable()
def InitAlgorithm(self, faceCasc, leftEyeCasc, rightEyeCasc, loadPreprocessedData, loadMLP):
# load pre-trained cascades
self.faceCasc = cv3.CascadeClassifier(faceCasc)
if self.faceCasc.empty():
print 'Warning: Could not load face cascade:', faceCasc
raise SystemExit
self.leftEyeCasc = cv3.CascadeClassifier(leftEyeCasc)
if self.leftEyeCasc.empty():
print 'Warning: Could not load left eye cascade:', leftEyeCasc
raise SystemExit
self.rightEyeCasc = cv3.CascadeClassifier(rightEyeCasc)
if self.rightEyeCasc.empty():
print 'Warning: Could not load right eye cascade:', rightEyeCasc
raise SystemExit
# load preprocessed dataset to access labels and PCA params
if path.isfile(loadPreprocessedData):
(_,y_train),(_,y_test),self.pca_V,self.pca_m = homebrew.load_from_file(loadPreprocessedData)
self.allLabels = np.unique(np.hstack((y_train,y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(loadMLP):
self.MLP = MultiLayerPerceptron(np.array([self.pca_V.shape[1], len(self.allLabels)]),
self.allLabels)
self.MLP.load(loadMLP)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", loadMLP
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find preprocessed data file ", loadPreprocessedData
self.testing.Disable()
def main():
# load training data
# training data can be recorded using chapter7.py in training mode
(X_train, y_train), (X_test, y_test), _, _ = homebrew.load_data(
"datasets/faces_training.pkl",
num_components=50,
test_split=0.2,
save_to_file="datasets/faces_preprocessed.pkl",
seed=42)
if len(X_train) == 0 or len(X_test) == 0:
print "Empty data"
raise SystemExit
# convert to numpy
X_train = np.squeeze(np.array(X_train)).astype(np.float32)
y_train = np.array(y_train)
X_test = np.squeeze(np.array(X_test)).astype(np.float32)
y_test = np.array(y_test)
# find all class labels
labels = np.unique(np.hstack((y_train, y_test)))
# prepare training
num_features = len(X_train[0])
num_classes = len(labels)
params = dict(term_crit=(cv2.TERM_CRITERIA_COUNT, 300, 0.01),
train_method=cv2.ANN_MLP_TRAIN_PARAMS_BACKPROP,
bp_dw_scale=0.001,
bp_moment_scale=0.9)
saveFile = 'params/mlp.xml'
# find best MLP configuration
print "---"
print "1-hidden layer networks"
best_acc = 0.0 # keep track of best accuracy
for l1 in xrange(10):
# gradually increase the hidden-layer size
layerSizes = np.int32(
[num_features, (l1 + 1) * num_features / 5, num_classes])
MLP = MultiLayerPerceptron(layerSizes, labels)
print layerSizes
MLP.fit(X_train, y_train, params=params)
(acc, _, _) = MLP.evaluate(X_train, y_train)
print " - train acc = ", acc
(acc, _, _) = MLP.evaluate(X_test, y_test)
print " - test acc = ", acc
if acc > best_acc:
# save best MLP configuration to file
MLP.save(saveFile)
best_acc = acc
def init_algorithm(
self,
save_training_file='datasets/faces_training.pkl',
load_preprocessed_data='datasets/faces_preprocessed.pkl',
load_mlp='params/mlp.xml',
face_casc='params/haarcascade_frontalface_default.xml',
left_eye_casc='params/haarcascade_lefteye_2splits.xml',
right_eye_casc='params/haarcascade_righteye_2splits.xml'):
"""Initializes face detector and facial expression classifier
This method initializes both the face detector and the facial
expression classifier.
:param save_training_file: filename for storing the assembled
training set
:param load_preprocessed_data: filename for loading a previously
preprocessed dataset (for
classification in Testing Mode)
:param load_mlp: filename for loading a pre-trained
MLP classifier (use the script
train_test_mlp.py)
:param face_casc: path to a face cascade
:param left_eye_casc: path to a left-eye cascade
:param right_eye_casc: path to a right-eye cascade
"""
self.data_file = save_training_file
self.faces = FaceDetector(face_casc, left_eye_casc, right_eye_casc)
self.head = None
# load preprocessed dataset to access labels and PCA params
if path.isfile(load_preprocessed_data):
(_, y_train), (_, y_test), V, m = homebrew.load_from_file(
load_preprocessed_data)
self.pca_V = V
self.pca_m = m
self.all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array([self.pca_V.shape[1],
len(self.all_labels)])
self.MLP = MultiLayerPerceptron(layer_sizes, self.all_labels)
self.MLP.load(load_mlp)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", load_mlp
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find data file ", load_preprocessed_data
self.testing.Disable()
def InitAlgorithm(self, faceCasc, leftEyeCasc, rightEyeCasc,
loadPreprocessedData, loadMLP):
# load pre-trained cascades
self.faceCasc = cv3.CascadeClassifier(faceCasc)
if self.faceCasc.empty():
print 'Warning: Could not load face cascade:', faceCasc
raise SystemExit
self.leftEyeCasc = cv3.CascadeClassifier(leftEyeCasc)
if self.leftEyeCasc.empty():
print 'Warning: Could not load left eye cascade:', leftEyeCasc
raise SystemExit
self.rightEyeCasc = cv3.CascadeClassifier(rightEyeCasc)
if self.rightEyeCasc.empty():
print 'Warning: Could not load right eye cascade:', rightEyeCasc
raise SystemExit
# load preprocessed dataset to access labels and PCA params
if path.isfile(loadPreprocessedData):
(_, y_train), (
_, y_test), self.pca_V, self.pca_m = homebrew.load_from_file(
loadPreprocessedData)
self.allLabels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(loadMLP):
self.MLP = MultiLayerPerceptron(
np.array([self.pca_V.shape[1],
len(self.allLabels)]), self.allLabels)
self.MLP.load(loadMLP)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", loadMLP
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find preprocessed data file ", loadPreprocessedData
self.testing.Disable()
def main():
(X_train, y_train), (X_test, y_test), _, _ = homebrew.load_data(
load_from_folder="/datasets",
num_components=50,
test_split=0.2,
save_to_file="datasets/faces_preprocessed.pkl",
seed=42)
if len(X_train) == 0 or len(X_test) == 0:
print "Empty data"
raise SystemExit
X_train = np.squeeze(np.array(X_train)).astype(np.float32)
y_train = np.array(y_train)
X_test = np.squeeze(np.array(X_test)).astype(np.float32)
y_test = np.array(y_test)
labels = np.unique(np.hstack((y_train, y_test)))
print X_train.shape
num_features = len(X_train[0])
print num_features
num_classes = len(labels)
print num_classes
params = dict(term_crit=(cv2.TERM_CRITERIA_COUNT, 300, 0.01),
train_method=cv2.ANN_MLP_TRAIN_PARAMS_BACKPROP,
bp_dw_scale=0.001,
bp_moment_scale=0.9)
saveFile = 'params/mlp.xml'
print "---"
print "1-hidden layer networks"
best_acc = 0.0
for l1 in xrange(20):
layerSizes = np.int32([
num_features, (l1 + 1) * num_features / 5,
(l1 + 1) * num_features / 10, num_classes
])
MLP = MultiLayerPerceptron(layerSizes, labels)
print layerSizes
MLP.fit(X_train, y_train, params=params)
(acc, _, _) = MLP.evaluate(X_train, y_train)
print " - train acc = ", acc
(acc, _, _) = MLP.evaluate(X_test, y_test)
print " - test acc = ", acc
if acc > best_acc:
MLP.save(saveFile)
best_acc = acc
def main():
# load training data
# training data can be recorded using chapter7.py in training mode
(X_train, y_train), (X_test, y_test), _, _ = homebrew.load_data(
"datasets/faces_training.pkl",
num_components=50,
test_split=0.2,
save_to_file="datasets/faces_preprocessed.pkl",
seed=42)
if len(X_train) == 0 or len(X_test) == 0:
print "Empty data"
raise SystemExit
# convert to numpy
X_train = np.squeeze(np.array(X_train)).astype(np.float32)
y_train = np.array(y_train)
X_test = np.squeeze(np.array(X_test)).astype(np.float32)
y_test = np.array(y_test)
# find all class labels
labels = np.unique(np.hstack((y_train, y_test)))
# prepare training
num_features = len(X_train[0])
num_classes = len(labels)
params = dict(term_crit=(cv2.TERM_CRITERIA_COUNT, 300, 0.01),
train_method=cv2.ANN_MLP_TRAIN_PARAMS_BACKPROP,
bp_dw_scale=0.001, bp_moment_scale=0.9)
saveFile = 'params/mlp.xml'
# find best MLP configuration
print "---"
print "1-hidden layer networks"
best_acc = 0.0 # keep track of best accuracy
for l1 in xrange(10):
# gradually increase the hidden-layer size
layerSizes = np.int32([num_features, (l1 + 1) * num_features/5,
num_classes])
MLP = MultiLayerPerceptron(layerSizes, labels)
print layerSizes
MLP.fit(X_train, y_train, params=params)
(acc, _, _) = MLP.evaluate(X_train, y_train)
print " - train acc = ", acc
(acc, _, _) = MLP.evaluate(X_test, y_test)
print " - test acc = ", acc
if acc > best_acc:
# save best MLP configuration to file
MLP.save(saveFile)
best_acc = acc
class FaceLayout(BaseLayout):
def _init_custom_layout(self):
# initialize data structure
self.samples = []
self.labels = []
# call method to save data upon exiting
self.Bind(wx.EVT_CLOSE, self._on_exit)
def init_algorithm(
self,
save_training_file='datasets/faces_training.pkl',
load_preprocessed_data='datasets/faces_preprocessed.pkl',
load_mlp='params/mlp.xml',
face_casc='params/haarcascade_frontalface_default.xml',
left_eye_casc='params/haarcascade_lefteye_2splits.xml',
right_eye_casc='params/haarcascade_righteye_2splits.xml'):
self.data_file = save_training_file
self.faces = FaceDetector(face_casc, left_eye_casc, right_eye_casc)
self.head = None
# load preprocessed dataset to access labels and PCA params
if path.isfile(load_preprocessed_data):
(_, y_train), (_, y_test), V, m = homebrew.load_from_file(
load_preprocessed_data)
self.pca_V = V
self.pca_m = m
self.all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array([self.pca_V.shape[1],
len(self.all_labels)])
self.MLP = MultiLayerPerceptron(layer_sizes, self.all_labels)
self.MLP.load(load_mlp)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", load_mlp
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find data file ", load_preprocessed_data
self.testing.Disable()
def _create_custom_layout(self):
# create horizontal layout with train/test buttons
pnl1 = wx.Panel(self, -1)
self.training = wx.RadioButton(pnl1, -1, 'Train', (10, 10),
style=wx.RB_GROUP)
self.Bind(wx.EVT_RADIOBUTTON, self._on_training, self.training)
self.testing = wx.RadioButton(pnl1, -1, 'Test')
self.Bind(wx.EVT_RADIOBUTTON, self._on_testing, self.testing)
hbox1 = wx.BoxSizer(wx.HORIZONTAL)
hbox1.Add(self.training, 1)
hbox1.Add(self.testing, 1)
pnl1.SetSizer(hbox1)
# create a horizontal layout with all buttons
pnl2 = wx.Panel(self, -1)
self.neutral = wx.RadioButton(pnl2, -1, 'neutral', (10, 10),
style=wx.RB_GROUP)
self.happy = wx.RadioButton(pnl2, -1, 'happy')
self.sad = wx.RadioButton(pnl2, -1, 'sad')
self.surprised = wx.RadioButton(pnl2, -1, 'surprised')
self.angry = wx.RadioButton(pnl2, -1, 'angry')
self.disgusted = wx.RadioButton(pnl2, -1, 'disgusted')
hbox2 = wx.BoxSizer(wx.HORIZONTAL)
hbox2.Add(self.neutral, 1)
hbox2.Add(self.happy, 1)
hbox2.Add(self.sad, 1)
hbox2.Add(self.surprised, 1)
hbox2.Add(self.angry, 1)
hbox2.Add(self.disgusted, 1)
pnl2.SetSizer(hbox2)
# create horizontal layout with single snapshot button
pnl3 = wx.Panel(self, -1)
self.snapshot = wx.Button(pnl3, -1, 'Take Snapshot')
self.Bind(wx.EVT_BUTTON, self._on_snapshot, self.snapshot)
hbox3 = wx.BoxSizer(wx.HORIZONTAL)
hbox3.Add(self.snapshot, 1)
pnl3.SetSizer(hbox3)
# arrange all horizontal layouts vertically
self.panels_vertical.Add(pnl1, flag=wx.EXPAND | wx.TOP, border=1)
self.panels_vertical.Add(pnl2, flag=wx.EXPAND | wx.BOTTOM, border=1)
self.panels_vertical.Add(pnl3, flag=wx.EXPAND | wx.BOTTOM, border=1)
def _process_frame(self, frame):
# detect face
success, frame, self.head, (x, y) = self.faces.detect(frame)
if success and self.testing.GetValue():
# if face found: preprocess (align)
success, head = self.faces.align_head(self.head)
if success:
# extract features using PCA (loaded from file)
X, _, _ = homebrew.extract_features([head.flatten()],
self.pca_V, self.pca_m)
# predict label with pre-trained MLP
label = self.MLP.predict(np.array(X))[0]
# draw label above bounding box
cv2.putText(frame, str(label), (x, y - 20),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
return frame
def _on_training(self, evt):
self.neutral.Enable()
self.happy.Enable()
self.sad.Enable()
self.surprised.Enable()
self.angry.Enable()
self.disgusted.Enable()
self.snapshot.Enable()
def _on_testing(self, evt):
self.neutral.Disable()
self.happy.Disable()
self.sad.Disable()
self.surprised.Disable()
self.angry.Disable()
self.disgusted.Disable()
self.snapshot.Disable()
def _on_snapshot(self, evt):
if self.neutral.GetValue():
label = 'neutral'
elif self.happy.GetValue():
label = 'happy'
elif self.sad.GetValue():
label = 'sad'
elif self.surprised.GetValue():
label = 'surprised'
elif self.angry.GetValue():
label = 'angry'
elif self.disgusted.GetValue():
label = 'disgusted'
if self.head is None:
print "No face detected"
else:
success, head = self.faces.align_head(self.head)
if success:
print "Added sample to training set"
self.samples.append(head.flatten())
self.labels.append(label)
else:
print "Could not align head (eye detection failed?)"
def _on_exit(self, evt):
# if we have collected some samples, dump them to file
if len(self.samples) > 0:
# make sure we don't overwrite an existing file
if path.isfile(self.data_file):
# file already exists, construct new load_from_file
load_from_file, fileext = path.splitext(self.data_file)
offset = 0
while True:
file = load_from_file + "-" + str(offset) + fileext
if path.isfile(file):
offset += 1
else:
break
self.data_file = file
# dump samples and labels to file
f = open(self.data_file, 'wb')
pickle.dump(self.samples, f)
pickle.dump(self.labels, f)
f.close()
# inform user that file was created
print "Saved", len(self.samples), "samples to", self.data_file
# deallocate
self.Destroy()
class FaceLayout(BaseLayout):
"""A custom layout for face detection and facial expression recognition
A GUI to both assemble a training set and to perform real-time
classification on the live stream of a webcam using a pre-trained
classifier.
The GUI operates in two different modes:
* Training Mode: In training mode, the app will collect image frames,
detect a face therein, assignassign a label depending
on the facial expression, and upon exiting save all
collected data samples in a file, so that it can be
parsed by datasets.homebrew.
* Testing Mode: In testing mode, the app will detect a face in each
video frame and predict the corresponding class
label using a pre-trained MLP.
"""
def _init_custom_layout(self):
"""Initializes GUI"""
# initialize data structure
self.samples = []
self.labels = []
# call method to save data upon exiting
self.Bind(wx.EVT_CLOSE, self._on_exit)
def init_algorithm(
self,
save_training_file='datasets/faces_training.pkl',
load_preprocessed_data='datasets/faces_preprocessed.pkl',
load_mlp='params/mlp.xml',
face_casc='params/haarcascade_frontalface_default.xml',
left_eye_casc='params/haarcascade_lefteye_2splits.xml',
right_eye_casc='params/haarcascade_righteye_2splits.xml'):
"""Initializes face detector and facial expression classifier
This method initializes both the face detector and the facial
expression classifier.
:param save_training_file: filename for storing the assembled
training set
:param load_preprocessed_data: filename for loading a previously
preprocessed dataset (for
classification in Testing Mode)
:param load_mlp: filename for loading a pre-trained
MLP classifier (use the script
train_test_mlp.py)
:param face_casc: path to a face cascade
:param left_eye_casc: path to a left-eye cascade
:param right_eye_casc: path to a right-eye cascade
"""
self.data_file = save_training_file
self.faces = FaceDetector(face_casc, left_eye_casc, right_eye_casc)
# load preprocessed dataset to access labels and PCA params
if path.isfile(load_preprocessed_data):
(_, y_train), (_, y_test), V, m = homebrew.load_from_file(
load_preprocessed_data)
self.pca_V = V
self.pca_m = m
self.all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array([self.pca_V.shape[1],
len(self.all_labels)])
self.MLP = MultiLayerPerceptron(layer_sizes, self.all_labels)
self.MLP.load(load_mlp)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", load_mlp
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find data file ", load_preprocessed_data
self.testing.Disable()
def _create_custom_layout(self):
"""Decorates the GUI with buttons for assigning class labels"""
# create horizontal layout with train/test buttons
pnl1 = wx.Panel(self, -1)
self.training = wx.RadioButton(pnl1, -1, 'Train', (10, 10),
style=wx.RB_GROUP)
self.Bind(wx.EVT_RADIOBUTTON, self._on_training, self.training)
self.testing = wx.RadioButton(pnl1, -1, 'Test')
self.Bind(wx.EVT_RADIOBUTTON, self._on_testing, self.testing)
hbox1 = wx.BoxSizer(wx.HORIZONTAL)
hbox1.Add(self.training, 1)
hbox1.Add(self.testing, 1)
pnl1.SetSizer(hbox1)
# create a horizontal layout with all buttons
pnl2 = wx.Panel(self, -1)
self.neutral = wx.RadioButton(pnl2, -1, 'neutral', (10, 10),
style=wx.RB_GROUP)
self.happy = wx.RadioButton(pnl2, -1, 'happy')
self.sad = wx.RadioButton(pnl2, -1, 'sad')
self.surprised = wx.RadioButton(pnl2, -1, 'surprised')
self.angry = wx.RadioButton(pnl2, -1, 'angry')
self.disgusted = wx.RadioButton(pnl2, -1, 'disgusted')
hbox2 = wx.BoxSizer(wx.HORIZONTAL)
hbox2.Add(self.neutral, 1)
hbox2.Add(self.happy, 1)
hbox2.Add(self.sad, 1)
hbox2.Add(self.surprised, 1)
hbox2.Add(self.angry, 1)
hbox2.Add(self.disgusted, 1)
pnl2.SetSizer(hbox2)
# create horizontal layout with single snapshot button
pnl3 = wx.Panel(self, -1)
self.snapshot = wx.Button(pnl3, -1, 'Take Snapshot')
self.Bind(wx.EVT_BUTTON, self._on_snapshot, self.snapshot)
hbox3 = wx.BoxSizer(wx.HORIZONTAL)
hbox3.Add(self.snapshot, 1)
pnl3.SetSizer(hbox3)
# arrange all horizontal layouts vertically
self.panels_vertical.Add(pnl1, flag=wx.EXPAND | wx.TOP, border=1)
self.panels_vertical.Add(pnl2, flag=wx.EXPAND | wx.BOTTOM, border=1)
self.panels_vertical.Add(pnl3, flag=wx.EXPAND | wx.BOTTOM, border=1)
def _process_frame(self, frame):
"""Processes each captured frame
This method processes each captured frame.
* Training mode: Performs face detection.
* Testing mode: Performs face detection, and predicts the class
label of the facial expression.
"""
# detect face
success, frame, self.head = self.faces.detect(frame)
if success and self.testing.GetValue():
# if face found: preprocess (align)
success, head = self.faces.align_head(self.head)
if success:
# extract features using PCA (loaded from file)
X, _, _ = homebrew.extract_features([head.flatten()],
self.pca_V, self.pca_m)
# predict label with pre-trained MLP
label = self.MLP.predict(np.array(X))[0]
# draw label above bounding box
cv2.putText(frame, str(label), (x, y - 20),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
return frame
def _on_training(self, evt):
"""Enables all training-related buttons when Training Mode is on"""
self.neutral.Enable()
self.happy.Enable()
self.sad.Enable()
self.surprised.Enable()
self.angry.Enable()
self.disgusted.Enable()
self.snapshot.Enable()
def _on_testing(self, evt):
"""Disables all training-related buttons when Testing Mode is on"""
self.neutral.Disable()
self.happy.Disable()
self.sad.Disable()
self.surprised.Disable()
self.angry.Disable()
self.disgusted.Disable()
self.snapshot.Disable()
def _on_snapshot(self, evt):
"""Takes a snapshot of the current frame
This method takes a snapshot of the current frame, preprocesses
it to extract the head region, and upon success adds the data
sample to the training set.
"""
if self.neutral.GetValue():
label = 'neutral'
elif self.happy.GetValue():
label = 'happy'
elif self.sad.GetValue():
label = 'sad'
elif self.surprised.GetValue():
label = 'surprised'
elif self.angry.GetValue():
label = 'angry'
elif self.disgusted.GetValue():
label = 'disgusted'
if self.head is None:
print "No face detected"
else:
success, head = self.faces.align_head(self.head)
if success:
print "Added sample to training set"
self.samples.append(head.flatten())
self.labels.append(label)
else:
print "Could not align head (eye detection failed?)"
def _on_exit(self, evt):
"""Dumps the training data to file upon exiting"""
# if we have collected some samples, dump them to file
if len(self.samples) > 0:
# make sure we don't overwrite an existing file
if path.isfile(self.data_file):
# file already exists, construct new load_from_file
load_from_file, fileext = path.splitext(self.data_file)
offset = 0
while True:
file = load_from_file + "-" + str(offset) + fileext
if path.isfile(file):
offset += 1
else:
break
self.data_file = file
# dump samples and labels to file
f = open(self.data_file, 'wb')
pickle.dump(self.samples, f)
pickle.dump(self.labels, f)
f.close()
# inform user that file was created
print "Saved", len(self.samples), "samples to", self.data_file
# deallocate
self.Destroy()
class FaceLayout(BaseLayout):
"""A custom layout for face detection and facial expression recognition
A GUI to both assemble a training set and to perform real-time
classification on the live stream of a webcam using a pre-trained
classifier.
The GUI operates in two different modes:
* Training Mode: In training mode, the app will collect image frames,
detect a face therein, assignassign a label depending
on the facial expression, and upon exiting save all
collected data samples in a file, so that it can be
parsed by datasets.homebrew.
* Testing Mode: In testing mode, the app will detect a face in each
video frame and predict the corresponding class
label using a pre-trained MLP.
"""
def _init_custom_layout(self):
"""Initializes GUI"""
# initialize data structure
self.samples = []
self.labels = []
# call method to save data upon exiting
self.Bind(wx.EVT_CLOSE, self._on_exit)
def init_algorithm(
self,
save_training_file='datasets/faces_training.pkl',
load_preprocessed_data='datasets/faces_preprocessed.pkl',
load_mlp='params/mlp.xml',
face_casc='params/haarcascade_frontalface_default.xml',
left_eye_casc='params/haarcascade_lefteye_2splits.xml',
right_eye_casc='params/haarcascade_righteye_2splits.xml'):
"""Initializes face detector and facial expression classifier
This method initializes both the face detector and the facial
expression classifier.
:param save_training_file: filename for storing the assembled
training set
:param load_preprocessed_data: filename for loading a previously
preprocessed dataset (for
classification in Testing Mode)
:param load_mlp: filename for loading a pre-trained
MLP classifier (use the script
train_test_mlp.py)
:param face_casc: path to a face cascade
:param left_eye_casc: path to a left-eye cascade
:param right_eye_casc: path to a right-eye cascade
"""
self.data_file = save_training_file
self.faces = FaceDetector(face_casc, left_eye_casc, right_eye_casc)
# load preprocessed dataset to access labels and PCA params
if path.isfile(load_preprocessed_data):
(_, y_train), (
_,
y_test), V, m = homebrew.load_from_file(load_preprocessed_data)
self.pca_V = V
self.pca_m = m
self.all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array(
[self.pca_V.shape[1],
len(self.all_labels)])
self.MLP = MultiLayerPerceptron(layer_sizes, self.all_labels)
self.MLP.load(load_mlp)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", load_mlp
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find data file ", load_preprocessed_data
self.testing.Disable()
def _create_custom_layout(self):
"""Decorates the GUI with buttons for assigning class labels"""
# create horizontal layout with train/test buttons
pnl1 = wx.Panel(self, -1)
self.training = wx.RadioButton(pnl1,
-1,
'Train', (10, 10),
style=wx.RB_GROUP)
self.Bind(wx.EVT_RADIOBUTTON, self._on_training, self.training)
self.testing = wx.RadioButton(pnl1, -1, 'Test')
self.Bind(wx.EVT_RADIOBUTTON, self._on_testing, self.testing)
hbox1 = wx.BoxSizer(wx.HORIZONTAL)
hbox1.Add(self.training, 1)
hbox1.Add(self.testing, 1)
pnl1.SetSizer(hbox1)
# create a horizontal layout with all buttons
pnl2 = wx.Panel(self, -1)
self.neutral = wx.RadioButton(pnl2,
-1,
'neutral', (10, 10),
style=wx.RB_GROUP)
self.happy = wx.RadioButton(pnl2, -1, 'happy')
self.sad = wx.RadioButton(pnl2, -1, 'sad')
self.surprised = wx.RadioButton(pnl2, -1, 'surprised')
self.angry = wx.RadioButton(pnl2, -1, 'angry')
self.disgusted = wx.RadioButton(pnl2, -1, 'disgusted')
hbox2 = wx.BoxSizer(wx.HORIZONTAL)
hbox2.Add(self.neutral, 1)
hbox2.Add(self.happy, 1)
hbox2.Add(self.sad, 1)
hbox2.Add(self.surprised, 1)
hbox2.Add(self.angry, 1)
hbox2.Add(self.disgusted, 1)
pnl2.SetSizer(hbox2)
# create horizontal layout with single snapshot button
pnl3 = wx.Panel(self, -1)
self.snapshot = wx.Button(pnl3, -1, 'Take Snapshot')
self.Bind(wx.EVT_BUTTON, self._on_snapshot, self.snapshot)
hbox3 = wx.BoxSizer(wx.HORIZONTAL)
hbox3.Add(self.snapshot, 1)
pnl3.SetSizer(hbox3)
# arrange all horizontal layouts vertically
self.panels_vertical.Add(pnl1, flag=wx.EXPAND | wx.TOP, border=1)
self.panels_vertical.Add(pnl2, flag=wx.EXPAND | wx.BOTTOM, border=1)
self.panels_vertical.Add(pnl3, flag=wx.EXPAND | wx.BOTTOM, border=1)
def _process_frame(self, frame):
"""Processes each captured frame
This method processes each captured frame.
* Training mode: Performs face detection.
* Testing mode: Performs face detection, and predicts the class
label of the facial expression.
"""
# detect face
success, frame, self.head = self.faces.detect(frame)
if success and self.testing.GetValue():
# if face found: preprocess (align)
success, head = self.faces.align_head(self.head)
if success:
# extract features using PCA (loaded from file)
X, _, _ = homebrew.extract_features([head.flatten()],
self.pca_V, self.pca_m)
# predict label with pre-trained MLP
label = self.MLP.predict(np.array(X))[0]
# draw label above bounding box
cv2.putText(frame, str(label), (x, y - 20),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
return frame
def _on_training(self, evt):
"""Enables all training-related buttons when Training Mode is on"""
self.neutral.Enable()
self.happy.Enable()
self.sad.Enable()
self.surprised.Enable()
self.angry.Enable()
self.disgusted.Enable()
self.snapshot.Enable()
def _on_testing(self, evt):
"""Disables all training-related buttons when Testing Mode is on"""
self.neutral.Disable()
self.happy.Disable()
self.sad.Disable()
self.surprised.Disable()
self.angry.Disable()
self.disgusted.Disable()
self.snapshot.Disable()
def _on_snapshot(self, evt):
"""Takes a snapshot of the current frame
This method takes a snapshot of the current frame, preprocesses
it to extract the head region, and upon success adds the data
sample to the training set.
"""
if self.neutral.GetValue():
label = 'neutral'
elif self.happy.GetValue():
label = 'happy'
elif self.sad.GetValue():
label = 'sad'
elif self.surprised.GetValue():
label = 'surprised'
elif self.angry.GetValue():
label = 'angry'
elif self.disgusted.GetValue():
label = 'disgusted'
if self.head is None:
print "No face detected"
else:
success, head = self.faces.align_head(self.head)
if success:
print "Added sample to training set"
self.samples.append(head.flatten())
self.labels.append(label)
else:
print "Could not align head (eye detection failed?)"
def _on_exit(self, evt):
"""Dumps the training data to file upon exiting"""
# if we have collected some samples, dump them to file
if len(self.samples) > 0:
# make sure we don't overwrite an existing file
if path.isfile(self.data_file):
# file already exists, construct new load_from_file
load_from_file, fileext = path.splitext(self.data_file)
offset = 0
while True:
file = load_from_file + "-" + str(offset) + fileext
if path.isfile(file):
offset += 1
else:
break
self.data_file = file
# dump samples and labels to file
f = open(self.data_file, 'wb')
pickle.dump(self.samples, f)
pickle.dump(self.labels, f)
f.close()
# inform user that file was created
print "Saved", len(self.samples), "samples to", self.data_file
# deallocate
self.Destroy()
class MyFrame(wx.Frame):
def __init__(self, parent, id, title, capture,
saveTrainingFile='datasets/faces_training.pkl',
loadPreprocessedData='datasets/faces_preprocessed.pkl',
loadMLP='params/mlp.xml',
faceCasc='params/haarcascade_frontalface_default.xml',
leftEyeCasc='params/haarcascade_lefteye_2splits.xml',
rightEyeCasc='params/haarcascade_righteye_2splits.xml',
fps=10):
# initialize screen capture
self.capture = capture
ret,frame = self.capture.read()
# determine window size and init wx.Frame
self.imgHeight,self.imgWidth = frame.shape[:2]
self.bmp = wx.BitmapFromBuffer(self.imgWidth, self.imgHeight, frame)
wx.Frame.__init__(self, parent, id, title, size=(self.imgWidth, self.imgHeight+60))
# set up periodic screen capture
self.timer = wx.Timer(self)
self.timer.Start(1000./fps)
self.Bind(wx.EVT_TIMER, self.NextFrame)
self.Bind(wx.EVT_PAINT, self.OnPaint)
# counteract flicker
def disable_event(*pargs,**kwargs):
pass
self.Bind(wx.EVT_ERASE_BACKGROUND, disable_event)
# initialize data structure
self.dataFile = saveTrainingFile
self.samples = []
self.labels = []
# call method to save data upon exiting
self.Bind(wx.EVT_CLOSE, self.OnExit)
# create the layout, which draws all buttons and
# connects events to class methods
self.CreateLayout()
self.InitAlgorithm(faceCasc, leftEyeCasc, rightEyeCasc, loadPreprocessedData, loadMLP)
def InitAlgorithm(self, faceCasc, leftEyeCasc, rightEyeCasc, loadPreprocessedData, loadMLP):
# load pre-trained cascades
self.faceCasc = cv3.CascadeClassifier(faceCasc)
if self.faceCasc.empty():
print 'Warning: Could not load face cascade:', faceCasc
raise SystemExit
self.leftEyeCasc = cv3.CascadeClassifier(leftEyeCasc)
if self.leftEyeCasc.empty():
print 'Warning: Could not load left eye cascade:', leftEyeCasc
raise SystemExit
self.rightEyeCasc = cv3.CascadeClassifier(rightEyeCasc)
if self.rightEyeCasc.empty():
print 'Warning: Could not load right eye cascade:', rightEyeCasc
raise SystemExit
# load preprocessed dataset to access labels and PCA params
if path.isfile(loadPreprocessedData):
(_,y_train),(_,y_test),self.pca_V,self.pca_m = homebrew.load_from_file(loadPreprocessedData)
self.allLabels = np.unique(np.hstack((y_train,y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(loadMLP):
self.MLP = MultiLayerPerceptron(np.array([self.pca_V.shape[1], len(self.allLabels)]),
self.allLabels)
self.MLP.load(loadMLP)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", loadMLP
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find preprocessed data file ", loadPreprocessedData
self.testing.Disable()
def CreateLayout(self):
self.pnl1 = wx.Panel(self, -1, size=(self.imgWidth,self.imgHeight))
self.pnl1.SetBackgroundColour(wx.BLACK)
# create horizontal layout with train/test buttons
pnl2 = wx.Panel(self, -1)
self.training = wx.RadioButton(pnl2, -1, 'Train', (10, 10), style=wx.RB_GROUP)
self.Bind(wx.EVT_RADIOBUTTON, self.OnTraining, self.training)
self.testing = wx.RadioButton(pnl2, -1, 'Test')
self.Bind(wx.EVT_RADIOBUTTON, self.OnTesting, self.testing)
hbox2 = wx.BoxSizer(wx.HORIZONTAL)
hbox2.Add(self.training, 1)
hbox2.Add(self.testing, 1)
pnl2.SetSizer(hbox2)
# create a horizontal layout with all buttons
pnl3 = wx.Panel(self, -1 )
self.neutral = wx.RadioButton(pnl3, -1, 'neutral', (10, 10), style=wx.RB_GROUP)
self.happy = wx.RadioButton(pnl3, -1, 'happy')
self.sad = wx.RadioButton(pnl3, -1, 'sad')
self.surprised = wx.RadioButton(pnl3, -1, 'surprised')
self.angry = wx.RadioButton(pnl3, -1, 'angry')
self.disgusted = wx.RadioButton(pnl3, -1, 'disgusted')
hbox3 = wx.BoxSizer(wx.HORIZONTAL)
hbox3.Add(self.neutral, 1)
hbox3.Add(self.happy, 1)
hbox3.Add(self.sad, 1)
hbox3.Add(self.surprised, 1)
hbox3.Add(self.angry, 1)
hbox3.Add(self.disgusted, 1)
pnl3.SetSizer(hbox3)
# create horizontal layout with single snapshot button
pnl4 = wx.Panel(self, -1)
self.snapshot = wx.Button(pnl4, -1, 'Take Snapshot')
self.Bind(wx.EVT_BUTTON, self.OnSnapshot, self.snapshot)
hbox4 = wx.BoxSizer(wx.HORIZONTAL)
hbox4.Add(self.snapshot, 1)
pnl4.SetSizer(hbox4)
# arrange all horizontal layouts vertically
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(self.pnl1, 1, flag=wx.EXPAND)
sizer.Add(pnl2, flag=wx.EXPAND | wx.TOP, border=1)
sizer.Add(pnl3, flag=wx.EXPAND | wx.BOTTOM, border=1)
sizer.Add(pnl4, flag=wx.EXPAND | wx.BOTTOM, border=1)
self.SetMinSize((self.imgWidth, self.imgHeight))
self.SetSizer(sizer)
self.Centre()
def NextFrame(self, event):
""" called whenever new frame is received, prepares frame for display """
ret, self.thisFrame = self.capture.read()
if ret:
# process frame
frame = cv3.cvtColor(self.thisFrame, cv3.COLOR_BGR2RGB)
frame = self.ProcessFrame(frame)
# update buffer and paint (EVT_PAINT triggered by Refresh)
self.bmp.CopyFromBuffer(frame)
self.Refresh(eraseBackground=False)
def ProcessFrame(self, frame):
""" detects face, predicts face label in testing mode """
# detect face
scale = 4
frameCasc = cv3.cvtColor(cv3.resize(frame, (0,0), fx=1./scale, fy=1./scale), cv3.COLOR_RGB2GRAY)
faces = self.faceCasc.detectMultiScale(frameCasc, scaleFactor=1.1, minNeighbors=3,
flags=cv3.cv.CV_HAAR_FIND_BIGGEST_OBJECT) * scale
# if face is found
for (x,y,w,h) in faces:
# extract head region from bounding box
cv3.rectangle(frame, (x, y), (x+w, y+h), (100, 255, 0), 2)
self.head = cv3.cvtColor(frame[y:y+h, x:x+w], cv3.COLOR_RGB2GRAY)
# in testing mode: predict label of facial expression
if self.testing.GetValue()==True:
# preprocess face: align
ret,head = self.AlignHead(self.head)
if ret:
# extract features using PCA (loaded from file)
X,_,_ = homebrew.extract_features([head.flatten()], self.pca_V, self.pca_m)
# predict label with pre-trained MLP
label = self.MLP.predict(np.array(X))[0]
# draw label above bounding box
cv3.putText(frame, str(label), (x,y-20), cv3.FONT_HERSHEY_COMPLEX, 1, (0,255,0), 2)
break # need only look at first, largest face
return frame
def AlignHead(self, head):
""" aligns a head region using affine transformations (rotation, scaling) """
height,width = head.shape[:2]
# detect left eye
leftEyeRegion = head[0.2*height:0.5*height, 0.1*width:0.5*width]
leftEye = self.leftEyeCasc.detectMultiScale(leftEyeRegion, scaleFactor=1.1, minNeighbors=3,
flags=cv3.cv.CV_HAAR_FIND_BIGGEST_OBJECT)
leftEyeCenter = None
for (xl,yl,wl,hl) in leftEye:
# find the center of the detected eye region
leftEyeCenter = np.array([0.1*width+xl+wl/2, 0.2*height+yl+hl/2])
break # need only look at first, largest eye
# detect right eye
rightEyeRegion = head[0.2*height:0.5*height, 0.5*width:0.9*width]
rightEye = self.rightEyeCasc.detectMultiScale(rightEyeRegion, scaleFactor=1.1, minNeighbors=3,
flags=cv3.cv.CV_HAAR_FIND_BIGGEST_OBJECT)
rightEyeCenter = None
for (xr,yr,wr,hr) in rightEye:
# find the center of the detected eye region
rightEyeCenter = np.array([0.5*width+xr+wr/2, 0.2*height+yr+hr/2])
break # need only look at first, largest eye
# need both eyes in order to align face
# else break here and report failure (False)
if leftEyeCenter is None or rightEyeCenter is None:
return (False, head)
# we want the eye to be at 25% of the width, and 20% of the height
# resulting image should be square (desiredImgWidth,desiredImgHeight)
desiredEyeX = 0.25
desiredEyeY = 0.2
desiredImgWidth = 200
desiredImgHeight = desiredImgWidth
# get center point between the two eyes and calculate angle
eyeCenter = (leftEyeCenter+rightEyeCenter)/2
eyeAngleDeg = np.arctan2(rightEyeCenter[1]-leftEyeCenter[1],
rightEyeCenter[0]-leftEyeCenter[0])*180.0/cv3.cv.CV_PI
# scale distance between eyes to desired length
eyeSizeScale = (1.0-desiredEyeX*2)*desiredImgWidth/np.linalg.norm(rightEyeCenter-leftEyeCenter)
# get rotation matrix
rotMat = cv3.getRotationMatrix2D(tuple(eyeCenter), eyeAngleDeg, eyeSizeScale)
# shift center of the eyes to be centered in the image
rotMat[0,2] += desiredImgWidth*0.5 - eyeCenter[0]
rotMat[1,2] += desiredEyeY*desiredImgHeight - eyeCenter[1]
# warp perspective to make eyes aligned on horizontal line and scaled to right size
res = cv3.warpAffine(head, rotMat, (desiredImgWidth,desiredImgWidth))
# return success
return (True,res)
def OnTraining(self, evt):
""" whenever training mode is selected, enable all training-related buttons """
self.neutral.Enable()
self.happy.Enable()
self.sad.Enable()
self.surprised.Enable()
self.angry.Enable()
self.disgusted.Enable()
self.snapshot.Enable()
def OnTesting(self, evt):
""" whenever testing mode is selected, disable all training-related buttons """
self.neutral.Disable()
self.happy.Disable()
self.sad.Disable()
self.surprised.Disable()
self.angry.Disable()
self.disgusted.Disable()
self.snapshot.Disable()
def OnPaint(self, evt):
""" called whenever new frame needs to be drawn """
# read and draw buffered bitmap
deviceContext = wx.BufferedPaintDC(self.pnl1)
deviceContext.DrawBitmap(self.bmp, 0, 0)
del deviceContext
def OnSnapshot(self, evt):
""" called whenever Take Snapshot button is clicked """
if self.neutral.GetValue()==True:
label = 'neutral'
elif self.happy.GetValue()==True:
label = 'happy'
elif self.sad.GetValue()==True:
label = 'sad'
elif self.surprised.GetValue()==True:
label = 'surprised'
elif self.angry.GetValue()==True:
label = 'angry'
elif self.disgusted.GetValue()==True:
label = 'disgusted'
ret,head = self.AlignHead(self.head)
if ret:
self.samples.append(head.flatten())
self.labels.append(label)
else:
print "Could not align head (eye detection failed?)"
def OnExit(self, evt):
""" called whenever window is closed """
# if we have collected some samples, dump them to file
if len(self.samples) > 0:
# make sure we don't overwrite an existing file
if path.isfile(self.dataFile):
# file already exists, construct new load_from_file
load_from_file,fileext = path.splitext(self.dataFile)
offset = 0
while True:
file = load_from_file + "-" + str(offset) + fileext
if path.isfile(file):
offset += 1
else:
break
self.dataFile = file
# dump samples and labels to file
f = open(self.dataFile,'wb')
pickle.dump(self.samples, f)
pickle.dump(self.labels, f)
f.close()
# inform user that file was created
print "Saved", len(self.samples), "samples to",self.dataFile
# deallocate
self.Destroy()
class MyFrame(wx.Frame):
def __init__(self,
parent,
id,
title,
capture,
saveTrainingFile='datasets/faces_training.pkl',
loadPreprocessedData='datasets/faces_preprocessed.pkl',
loadMLP='params/mlp.xml',
faceCasc='params/haarcascade_frontalface_default.xml',
leftEyeCasc='params/haarcascade_lefteye_2splits.xml',
rightEyeCasc='params/haarcascade_righteye_2splits.xml',
fps=10):
# initialize screen capture
self.capture = capture
ret, frame = self.capture.read()
# determine window size and init wx.Frame
self.imgHeight, self.imgWidth = frame.shape[:2]
self.bmp = wx.BitmapFromBuffer(self.imgWidth, self.imgHeight, frame)
wx.Frame.__init__(self,
parent,
id,
title,
size=(self.imgWidth, self.imgHeight + 60))
# set up periodic screen capture
self.timer = wx.Timer(self)
self.timer.Start(1000. / fps)
self.Bind(wx.EVT_TIMER, self.NextFrame)
self.Bind(wx.EVT_PAINT, self.OnPaint)
# counteract flicker
def disable_event(*pargs, **kwargs):
pass
self.Bind(wx.EVT_ERASE_BACKGROUND, disable_event)
# initialize data structure
self.dataFile = saveTrainingFile
self.samples = []
self.labels = []
# call method to save data upon exiting
self.Bind(wx.EVT_CLOSE, self.OnExit)
# create the layout, which draws all buttons and
# connects events to class methods
self.CreateLayout()
self.InitAlgorithm(faceCasc, leftEyeCasc, rightEyeCasc,
loadPreprocessedData, loadMLP)
def InitAlgorithm(self, faceCasc, leftEyeCasc, rightEyeCasc,
loadPreprocessedData, loadMLP):
# load pre-trained cascades
self.faceCasc = cv3.CascadeClassifier(faceCasc)
if self.faceCasc.empty():
print 'Warning: Could not load face cascade:', faceCasc
raise SystemExit
self.leftEyeCasc = cv3.CascadeClassifier(leftEyeCasc)
if self.leftEyeCasc.empty():
print 'Warning: Could not load left eye cascade:', leftEyeCasc
raise SystemExit
self.rightEyeCasc = cv3.CascadeClassifier(rightEyeCasc)
if self.rightEyeCasc.empty():
print 'Warning: Could not load right eye cascade:', rightEyeCasc
raise SystemExit
# load preprocessed dataset to access labels and PCA params
if path.isfile(loadPreprocessedData):
(_, y_train), (
_, y_test), self.pca_V, self.pca_m = homebrew.load_from_file(
loadPreprocessedData)
self.allLabels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(loadMLP):
self.MLP = MultiLayerPerceptron(
np.array([self.pca_V.shape[1],
len(self.allLabels)]), self.allLabels)
self.MLP.load(loadMLP)
else:
print "Warning: Testing is disabled"
print "Could not find pre-trained MLP file ", loadMLP
self.testing.Disable()
else:
print "Warning: Testing is disabled"
print "Could not find preprocessed data file ", loadPreprocessedData
self.testing.Disable()
def CreateLayout(self):
self.pnl1 = wx.Panel(self, -1, size=(self.imgWidth, self.imgHeight))
self.pnl1.SetBackgroundColour(wx.BLACK)
# create horizontal layout with train/test buttons
pnl2 = wx.Panel(self, -1)
self.training = wx.RadioButton(pnl2,
-1,
'Train', (10, 10),
style=wx.RB_GROUP)
self.Bind(wx.EVT_RADIOBUTTON, self.OnTraining, self.training)
self.testing = wx.RadioButton(pnl2, -1, 'Test')
self.Bind(wx.EVT_RADIOBUTTON, self.OnTesting, self.testing)
hbox2 = wx.BoxSizer(wx.HORIZONTAL)
hbox2.Add(self.training, 1)
hbox2.Add(self.testing, 1)
pnl2.SetSizer(hbox2)
# create a horizontal layout with all buttons
pnl3 = wx.Panel(self, -1)
self.neutral = wx.RadioButton(pnl3,
-1,
'neutral', (10, 10),
style=wx.RB_GROUP)
self.happy = wx.RadioButton(pnl3, -1, 'happy')
self.sad = wx.RadioButton(pnl3, -1, 'sad')
self.surprised = wx.RadioButton(pnl3, -1, 'surprised')
self.angry = wx.RadioButton(pnl3, -1, 'angry')
self.disgusted = wx.RadioButton(pnl3, -1, 'disgusted')
hbox3 = wx.BoxSizer(wx.HORIZONTAL)
hbox3.Add(self.neutral, 1)
hbox3.Add(self.happy, 1)
hbox3.Add(self.sad, 1)
hbox3.Add(self.surprised, 1)
hbox3.Add(self.angry, 1)
hbox3.Add(self.disgusted, 1)
pnl3.SetSizer(hbox3)
# create horizontal layout with single snapshot button
pnl4 = wx.Panel(self, -1)
self.snapshot = wx.Button(pnl4, -1, 'Take Snapshot')
self.Bind(wx.EVT_BUTTON, self.OnSnapshot, self.snapshot)
hbox4 = wx.BoxSizer(wx.HORIZONTAL)
hbox4.Add(self.snapshot, 1)
pnl4.SetSizer(hbox4)
# arrange all horizontal layouts vertically
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(self.pnl1, 1, flag=wx.EXPAND)
sizer.Add(pnl2, flag=wx.EXPAND | wx.TOP, border=1)
sizer.Add(pnl3, flag=wx.EXPAND | wx.BOTTOM, border=1)
sizer.Add(pnl4, flag=wx.EXPAND | wx.BOTTOM, border=1)
self.SetMinSize((self.imgWidth, self.imgHeight))
self.SetSizer(sizer)
self.Centre()
def NextFrame(self, event):
""" called whenever new frame is received, prepares frame for display """
ret, self.thisFrame = self.capture.read()
if ret:
# process frame
frame = cv3.cvtColor(self.thisFrame, cv3.COLOR_BGR2RGB)
frame = self.ProcessFrame(frame)
# update buffer and paint (EVT_PAINT triggered by Refresh)
self.bmp.CopyFromBuffer(frame)
self.Refresh(eraseBackground=False)
def ProcessFrame(self, frame):
""" detects face, predicts face label in testing mode """
# detect face
scale = 4
frameCasc = cv3.cvtColor(
cv3.resize(frame, (0, 0), fx=1. / scale, fy=1. / scale),
cv3.COLOR_RGB2GRAY)
faces = self.faceCasc.detectMultiScale(
frameCasc,
scaleFactor=1.1,
minNeighbors=3,
flags=cv3.cv.CV_HAAR_FIND_BIGGEST_OBJECT) * scale
# if face is found
for (x, y, w, h) in faces:
# extract head region from bounding box
cv3.rectangle(frame, (x, y), (x + w, y + h), (100, 255, 0), 2)
self.head = cv3.cvtColor(frame[y:y + h, x:x + w],
cv3.COLOR_RGB2GRAY)
# in testing mode: predict label of facial expression
if self.testing.GetValue() == True:
# preprocess face: align
ret, head = self.AlignHead(self.head)
if ret:
# extract features using PCA (loaded from file)
X, _, _ = homebrew.extract_features([head.flatten()],
self.pca_V, self.pca_m)
# predict label with pre-trained MLP
label = self.MLP.predict(np.array(X))[0]
# draw label above bounding box
cv3.putText(frame, str(label), (x, y - 20),
cv3.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
break # need only look at first, largest face
return frame
def AlignHead(self, head):
""" aligns a head region using affine transformations (rotation, scaling) """
height, width = head.shape[:2]
# detect left eye
leftEyeRegion = head[0.2 * height:0.5 * height,
0.1 * width:0.5 * width]
leftEye = self.leftEyeCasc.detectMultiScale(
leftEyeRegion,
scaleFactor=1.1,
minNeighbors=3,
flags=cv3.cv.CV_HAAR_FIND_BIGGEST_OBJECT)
leftEyeCenter = None
for (xl, yl, wl, hl) in leftEye:
# find the center of the detected eye region
leftEyeCenter = np.array(
[0.1 * width + xl + wl / 2, 0.2 * height + yl + hl / 2])
break # need only look at first, largest eye
# detect right eye
rightEyeRegion = head[0.2 * height:0.5 * height,
0.5 * width:0.9 * width]
rightEye = self.rightEyeCasc.detectMultiScale(
rightEyeRegion,
scaleFactor=1.1,
minNeighbors=3,
flags=cv3.cv.CV_HAAR_FIND_BIGGEST_OBJECT)
rightEyeCenter = None
for (xr, yr, wr, hr) in rightEye:
# find the center of the detected eye region
rightEyeCenter = np.array(
[0.5 * width + xr + wr / 2, 0.2 * height + yr + hr / 2])
break # need only look at first, largest eye
# need both eyes in order to align face
# else break here and report failure (False)
if leftEyeCenter is None or rightEyeCenter is None:
return (False, head)
# we want the eye to be at 25% of the width, and 20% of the height
# resulting image should be square (desiredImgWidth,desiredImgHeight)
desiredEyeX = 0.25
desiredEyeY = 0.2
desiredImgWidth = 200
desiredImgHeight = desiredImgWidth
# get center point between the two eyes and calculate angle
eyeCenter = (leftEyeCenter + rightEyeCenter) / 2
eyeAngleDeg = np.arctan2(
rightEyeCenter[1] - leftEyeCenter[1],
rightEyeCenter[0] - leftEyeCenter[0]) * 180.0 / cv3.cv.CV_PI
# scale distance between eyes to desired length
eyeSizeScale = (1.0 - desiredEyeX *
2) * desiredImgWidth / np.linalg.norm(rightEyeCenter -
leftEyeCenter)
# get rotation matrix
rotMat = cv3.getRotationMatrix2D(tuple(eyeCenter), eyeAngleDeg,
eyeSizeScale)
# shift center of the eyes to be centered in the image
rotMat[0, 2] += desiredImgWidth * 0.5 - eyeCenter[0]
rotMat[1, 2] += desiredEyeY * desiredImgHeight - eyeCenter[1]
# warp perspective to make eyes aligned on horizontal line and scaled to right size
res = cv3.warpAffine(head, rotMat, (desiredImgWidth, desiredImgWidth))
# return success
return (True, res)
def OnTraining(self, evt):
""" whenever training mode is selected, enable all training-related buttons """
self.neutral.Enable()
self.happy.Enable()
self.sad.Enable()
self.surprised.Enable()
self.angry.Enable()
self.disgusted.Enable()
self.snapshot.Enable()
def OnTesting(self, evt):
""" whenever testing mode is selected, disable all training-related buttons """
self.neutral.Disable()
self.happy.Disable()
self.sad.Disable()
self.surprised.Disable()
self.angry.Disable()
self.disgusted.Disable()
self.snapshot.Disable()
def OnPaint(self, evt):
""" called whenever new frame needs to be drawn """
# read and draw buffered bitmap
deviceContext = wx.BufferedPaintDC(self.pnl1)
deviceContext.DrawBitmap(self.bmp, 0, 0)
del deviceContext
def OnSnapshot(self, evt):
""" called whenever Take Snapshot button is clicked """
if self.neutral.GetValue() == True:
label = 'neutral'
elif self.happy.GetValue() == True:
label = 'happy'
elif self.sad.GetValue() == True:
label = 'sad'
elif self.surprised.GetValue() == True:
label = 'surprised'
elif self.angry.GetValue() == True:
label = 'angry'
elif self.disgusted.GetValue() == True:
label = 'disgusted'
ret, head = self.AlignHead(self.head)
if ret:
self.samples.append(head.flatten())
self.labels.append(label)
else:
print "Could not align head (eye detection failed?)"
def OnExit(self, evt):
""" called whenever window is closed """
# if we have collected some samples, dump them to file
if len(self.samples) > 0:
# make sure we don't overwrite an existing file
if path.isfile(self.dataFile):
# file already exists, construct new load_from_file
load_from_file, fileext = path.splitext(self.dataFile)
offset = 0
while True:
file = load_from_file + "-" + str(offset) + fileext
if path.isfile(file):
offset += 1
else:
break
self.dataFile = file
# dump samples and labels to file
f = open(self.dataFile, 'wb')
pickle.dump(self.samples, f)
pickle.dump(self.labels, f)
f.close()
# inform user that file was created
print "Saved", len(self.samples), "samples to", self.dataFile
# deallocate
self.Destroy()
load_mlp = 'params/mlp.xml'
light1 = 0
light2 = 0
if path.isfile(load_preprocessed_data):
(_,
y_train), (_,
y_test), V, m = homebrew.load_from_file(load_preprocessed_data)
pca_V = V
pca_m = m
all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array([pca_V.shape[1], len(all_labels)])
MLP = MultiLayerPerceptron(layer_sizes, all_labels)
MLP.load(load_mlp)
#############################################################################
import cv2
import numpy as np
import copy
import math
# Environment:
# OS : Mac OS EL Capitan
# python: 3.5
# opencv: 2.4.13
def single_play():
# parameters
load_mlp = 'params/mlp.xml'
light1 = 0
light2 = 0
if path.isfile(load_preprocessed_data):
(_,
y_train), (_,
y_test), V, m = homebrew.load_from_file(load_preprocessed_data)
pca_V = V
pca_m = m
all_labels = np.unique(np.hstack((y_train, y_test)))
# load pre-trained multi-layer perceptron
if path.isfile(load_mlp):
layer_sizes = np.array([pca_V.shape[1], len(all_labels)])
MLP = MultiLayerPerceptron(layer_sizes, all_labels)
MLP.load(load_mlp)
#############################################################################
import cv2
import numpy as np
import copy
import math
# Environment:
# OS : Mac OS EL Capitan
# python: 3.5
# opencv: 2.4.13
# parameters
cap_region_x_begin = 0.5 # start point/total width