def __init__(self):
# initializebevideo stream
self.video_stream = cv2.VideoCapture(
VIDEO_PREDICTOR.camera_source)
self.face_detector = cv2.CascadeClassifier(
VIDEO_PREDICTOR.face_detection_classifier)
self.shape_predictor = None
self.shape_predictor = dlib.shape_predictor(
DATASET.shape_predictor_path)
model = None
with tf.Graph().as_default():
network = build_model(
use_landmarks=True,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=True)
model = DNN(network)
if os.path.isfile(
TRAINING.save_model_path_landmarks_hog_sw):
model.load(TRAINING.save_model_path_landmarks_hog_sw)
else:
print("Error: file '{}' not found".format(
TRAINING.save_model_path_landmarks_hog_sw))
self.model = model
self.last_predicted_time = 0
self.last_predicted_confidence = 0
self.last_predicted_emotion = ""
def load_model():
model = None
with tf.Graph().as_default():
print("loading pretrained model...")
network = build_model(use_landmarks=True,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=True)
model = DNN(network)
if os.path.isfile(TRAINING.save_model_path_landmarks_hog_sw):
model.load(TRAINING.save_model_path_landmarks_hog_sw)
else:
print("Error: file '{}' not found".format(
TRAINING.save_model_path_landmarks_hog_sw))
return model
def LoadTable(self):
# SVM
def evaluate(model, X, Y):
predicted_Y = model.predict(X)
accuracy = accuracy_score(Y, predicted_Y)
return accuracy
def evaluate_cnn(model, X, X2, Y, use_landmarks=False):
if use_landmarks:
accuracy = model.evaluate([X, X2], Y)
else:
accuracy = model.evaluate(X, Y)
return accuracy[0]
lstfeature = ["landmarks", "landmarks_and_hog", "landmarks_and_hog_sw"]
for feature in lstfeature:
model = None
with tf.Graph().as_default():
print("loading pretrained model...")
if (feature == "landmarks_and_hog"):
data, validation, test = load_data_svm(validation=True,
test=True,
feature=feature)
if os.path.isfile(
TRAININGSVM.save_model_path_landmarks_hog):
with open(TRAININGSVM.save_model_path_landmarks_hog,
'rb') as f:
model = cPickle.load(f)
else:
print("Error: file '{}' not found".format(
TRAININGSVM.save_model_path_landmarks_hog))
print("--")
print("Validation samples landmarks_and_hog: {}".format(
len(validation['Y'])))
print("Test samples landmarks_and_hog: {}".format(
len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate(model, validation['X'],
validation['Y'])
print(" - validation accuracy = {0:.1f}".format(
validation_accuracy * 100))
test_accuracy = evaluate(model, test['X'], test['Y'])
print(" - test accuracy = {0:.1f}".format(test_accuracy *
100))
print(
" - evalution time = {0:.1f} sec".format(time.time() -
start_time))
PARAMETERINPUT.Validation_faceLandmarksHoG_svm = "{0:.1f}".format(
validation_accuracy * 100)
PARAMETERINPUT.Test_faceLandmarksHoG_svm = "{0:.1f}".format(
test_accuracy * 100)
PARAMETERINPUT.Time_faceLandmarksHoG_svm = "{0:.1f}".format(
time.time() - start_time)
elif (feature == "landmarks_and_hog_sw"):
data, validation, test = load_data_svm(validation=True,
test=True,
feature=feature)
if os.path.isfile(
TRAININGSVM.save_model_path_landmarks_hog_sw):
with open(TRAININGSVM.save_model_path_landmarks_hog_sw,
'rb') as f:
model = cPickle.load(f)
else:
print("Error: file '{}' not found".format(
TRAININGSVM.save_model_path_landmarks_hog_sw))
print("--")
print("Validation samples landmarks_and_hog_sw: {}".format(
len(validation['Y'])))
print("Test samples landmarks_and_hog_sw: {}".format(
len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate(model, validation['X'],
validation['Y'])
print(" - validation accuracy = {0:.1f}".format(
validation_accuracy * 100))
test_accuracy = evaluate(model, test['X'], test['Y'])
print(" - test accuracy = {0:.1f}".format(test_accuracy *
100))
print(
" - evalution time = {0:.1f} sec".format(time.time() -
start_time))
PARAMETERINPUT.Validation_faceLandmarksHoGSlidingWindow_svm = "{0:.1f}".format(
validation_accuracy * 100)
PARAMETERINPUT.Test_faceLandmarksHoGSlidingWindow_svm = "{0:.1f}".format(
test_accuracy * 100)
PARAMETERINPUT.Time_faceLandmarksHoGSlidingWindow_svm = "{0:.1f}".format(
time.time() - start_time)
else:
data, validation, test = load_data_svm(validation=True,
test=True,
feature=feature)
if os.path.isfile(TRAININGSVM.save_model_path_landmarks):
with open(TRAININGSVM.save_model_path_landmarks,
'rb') as f:
model = cPickle.load(f)
else:
print("Error: file '{}' not found".format(
TRAININGSVM.save_model_path_landmarks))
print("--")
print("Validation samples landmarks: {}".format(
len(validation['Y'])))
print("Test samples landmarks: {}".format(len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate(model, validation['X'],
validation['Y'])
print(" - validation accuracy = {0:.1f}".format(
validation_accuracy * 100))
test_accuracy = evaluate(model, test['X'], test['Y'])
print(" - test accuracy = {0:.1f}".format(test_accuracy *
100))
print(
" - evalution time = {0:.1f} sec".format(time.time() -
start_time))
PARAMETERINPUT.Validation_faceLandmarks_svm = "{0:.1f}".format(
validation_accuracy * 100)
PARAMETERINPUT.Test_faceLandmarks_svm = "{0:.1f}".format(
test_accuracy * 100)
PARAMETERINPUT.Time_faceLandmarks_svm = "{0:.1f}".format(
time.time() - start_time)
#CNN
lstfeature = [
"landmarks", "landmarks_and_hog", "raw", "landmarks_and_hog_sw"
]
for feature in lstfeature:
model = None
with tf.Graph().as_default():
print("loading pretrained model...")
if (feature == "landmarks_and_hog"):
data, validation, test = load_data_cnn(
validation=True,
test=True,
use_landmarks=False,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=False)
network = build_model(
use_landmarks=True,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=False)
model = DNN(network)
if os.path.isfile(TRAINING.save_model_path_landmarks_hog):
model.load(TRAINING.save_model_path_landmarks_hog)
else:
print("Error: file '{}' not found".format(
TRAINING.save_model_path_landmarks_hog))
print("--")
print("Validation samples: {}".format(len(
validation['Y'])))
print("Test samples: {}".format(len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate_cnn(model,
validation['X'],
validation['X2'],
validation['Y'],
use_landmarks=True)
print(
" - validation accuracy landmarks_and_hog = {0:.1f}".
format(validation_accuracy * 100))
test_accuracy = evaluate_cnn(model,
test['X'],
test['X2'],
test['Y'],
use_landmarks=True)
print(
" - test accuracy landmarks_and_hog = {0:.1f}".format(
test_accuracy * 100))
print(" - evalution time landmarks_and_hog = {0:.1f} sec".
format(time.time() - start_time))
PARAMETERINPUT.Validation_faceLandmarksHoG_cnn = "{0:.1f}".format(
validation_accuracy * 100)
PARAMETERINPUT.Test_faceLandmarksHoG_cnn = "{0:.1f}".format(
test_accuracy * 100)
PARAMETERINPUT.Time_faceLandmarksHoG_cnn = "{0:.1f}".format(
time.time() - start_time)
elif (feature == "landmarks_and_hog_sw"):
data, validation, test = load_data_cnn(
validation=True,
test=True,
use_landmarks=True,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=True)
network = build_model(
use_landmarks=True,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=True)
model = DNN(network)
if os.path.isfile(
TRAINING.save_model_path_landmarks_hog_sw):
model.load(TRAINING.save_model_path_landmarks_hog_sw)
else:
print("Error: file '{}' not found".format(
TRAINING.save_model_path_landmarks_hog_sw))
print("--")
print("Validation samples: {}".format(len(
validation['Y'])))
print("Test samples: {}".format(len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate_cnn(model,
validation['X'],
validation['X2'],
validation['Y'],
use_landmarks=True)
print(
" - validation accuracy landmarks_and_hog_sw = {0:.1f}"
.format(validation_accuracy * 100))
test_accuracy = evaluate_cnn(model,
test['X'],
test['X2'],
test['Y'],
use_landmarks=True)
print(" - test accuracy landmarks_and_hog_sw = {0:.1f}".
format(test_accuracy * 100))
print(
" - evalution time landmarks_and_hog_sw = {0:.1f} sec"
.format(time.time() - start_time))
PARAMETERINPUT.Validation_faceLandmarksHoGSlidingWindow_cnn = "{0:.1f}".format(
validation_accuracy * 100)
PARAMETERINPUT.Test_faceLandmarksHoGSlidingWindow_cnn = "{0:.1f}".format(
test_accuracy * 100)
PARAMETERINPUT.Time_faceLandmarksHoGSlidingWindow_cnn = "{0:.1f}".format(
time.time() - start_time)
elif (feature == "landmarks"):
data, validation, test = load_data_cnn(
validation=True,
test=True,
use_landmarks=True,
use_hog_and_landmarks=False,
use_hog_sliding_window_and_landmarks=False)
network = build_model(
use_landmarks=True,
use_hog_and_landmarks=False,
use_hog_sliding_window_and_landmarks=False)
model = DNN(network)
if os.path.isfile(TRAINING.save_model_path_landmarks):
model.load(TRAINING.save_model_path_landmarks)
else:
print("Error: file '{}' not found".format(
TRAINING.save_model_path_landmarks))
print("--")
print("Validation samples: {}".format(len(
validation['Y'])))
print("Test samples: {}".format(len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate_cnn(model,
validation['X'],
validation['X2'],
validation['Y'],
use_landmarks=True)
print(" - validation accuracy landmarks = {0:.1f}".format(
validation_accuracy * 100))
test_accuracy = evaluate_cnn(model,
test['X'],
test['X2'],
test['Y'],
use_landmarks=True)
print(" - test accuracy landmarks = {0:.1f}".format(
test_accuracy * 100))
print(" - evalution time landmarks = {0:.1f} sec".format(
time.time() - start_time))
PARAMETERINPUT.Validation_faceLandmarks_cnn = "{0:.1f}".format(
validation_accuracy * 100)
PARAMETERINPUT.Test_faceLandmarks_cnn = "{0:.1f}".format(
test_accuracy * 100)
PARAMETERINPUT.Time_faceLandmarks_cnn = "{0:.1f}".format(
time.time() - start_time)
else:
data, validation, test = load_data(validation=True,
test=True)
network = build_model()
model = DNN(network)
# Testing phase : load saved model and evaluate on test dataset
print("start evaluation...")
print("loading pretrained model...")
if os.path.isfile(TRAINING.save_model_path_raw):
model.load(TRAINING.save_model_path_raw)
else:
print("Error: file '{}' not found".format(
TRAINING.save_model_path_raw))
exit()
if not NETWORK.use_landmarks:
validation['X2'] = None
test['X2'] = None
print("--")
print("Validation samples: {}".format(len(
validation['Y'])))
print("Test samples: {}".format(len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate_cnn(model, validation['X'],
validation['X2'],
validation['Y'])
print(" - validation accuracy raw = {0:.1f}".format(
validation_accuracy * 100))
test_accuracy = evaluate_cnn(model, test['X'], test['X2'],
test['Y'])
print(" - test accuracy raw = {0:.1f}".format(
test_accuracy * 100))
print(" - evalution time raw = {0:.1f} sec".format(
time.time() - start_time))
PARAMETERINPUT.Validation_raw_cnn = "{0:.1f}".format(
validation_accuracy * 100)
PARAMETERINPUT.Test_raw_cnn = "{0:.1f}".format(
test_accuracy * 100)
PARAMETERINPUT.Time_raw_cnn = "{0:.1f}".format(
time.time() - start_time)
self.treeview.insert(
'',
'end',
text="CNN",
values=(
PARAMETERINPUT.Validation_raw_cnn,
PARAMETERINPUT.Validation_faceLandmarks_cnn,
PARAMETERINPUT.Validation_faceLandmarksHoG_cnn,
PARAMETERINPUT.Validation_faceLandmarksHoGSlidingWindow_cnn))
self.treeview.insert(
'',
'end',
text="SVM",
values=(
'----', PARAMETERINPUT.Validation_faceLandmarks_svm,
PARAMETERINPUT.Validation_faceLandmarksHoG_svm,
PARAMETERINPUT.Validation_faceLandmarksHoGSlidingWindow_svm))
lstfeature = ["landmarks", "landmarks_and_hog", "raw", "landmarks_and_hog_sw"]
for feature in lstfeature:
model = None
with tf.Graph().as_default():
print("loading pretrained model...")
if (feature == "landmarks_and_hog"):
data, validation, test = load_data_cnn(
validation=True,
test=True,
use_landmarks=False,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=False)
network = build_model(use_landmarks=True,
use_hog_and_landmarks=True,
use_hog_sliding_window_and_landmarks=False)
model = DNN(network)
if os.path.isfile(TRAINING.save_model_path_landmarks_hog):
model.load(TRAINING.save_model_path_landmarks_hog)
else:
print("Error: file '{}' not found".format(
TRAINING.save_model_path_landmarks_hog))
print("--")
print("Validation samples: {}".format(len(validation['Y'])))
print("Test samples: {}".format(len(test['Y'])))
print("--")
print("evaluating...")
start_time = time.time()
validation_accuracy = evaluate(model,